Merge branch 'RESTAPI-1057-enable-recursive-ls' into 'master'

Restapi 1057 enable recursive ls

See merge request firecrest/firecrest!288

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add the endpoints `/compute/nodes` and `/compute/nodes/{nodeName}` to retrieve information about nodes in the scheduling queue.
 - Added endpoints `POST /utilities/compress`, `POST /utilities/extract`, `POST /storage/xfer-internal/compress` and `POST /storage/xfer-internal/extract` for file compression and extraction.
+- Added recurisive option to ls utilities command `&recursive=true`
 
 ### Changed
 

doc/openapi/firecrest-api.yaml

@@ -207,6 +207,12 @@ paths:
           schema:
             type: boolean
             default: false
+        - name: recursive
+          in: query
+          description: List files recursively
+          schema:
+            type: boolean
+            default: false
         # - $ref: '#/components/parameters/pageSize'
         # - $ref: '#/components/parameters/pageNumber'
       responses:

doc/openapi/firecrest-developers-api.yaml

@@ -195,6 +195,12 @@ paths:
           schema:
             type: boolean
             default: false
+        - name: recursive
+          in: query
+          description: List files recursively
+          schema:
+            type: boolean
+            default: false
         # - $ref: '#/components/parameters/pageSize'
         # - $ref: '#/components/parameters/pageNumber'
       responses:

src/tests/automated_tests/unit/test_unit_utilities.py

@@ -245,6 +245,17 @@ def test_list_directory(machine, targetPath, expected_response_code, headers):
 	print(resp.headers)
 	assert resp.status_code == expected_response_code
 
+@skipif_not_uses_gateway
+@pytest.mark.parametrize("machine, targetPath, expected_response_code", DATA_LS)
+def test_list_directory_recursive(machine, targetPath, expected_response_code, headers):
+	params = {"targetPath": targetPath, "recursive" : "true"}
+	url = f"{UTILITIES_URL}/ls"
+	headers.update({"X-Machine-Name": machine})
+	resp = requests.get(url, headers=headers, params=params, verify= (f"{SSL_PATH}{SSL_CRT}" if USE_SSL else False))
+	print(json.dumps(resp.json(),indent=2))
+	print(resp.headers)
+	assert resp.status_code == expected_response_code
+
 
 @skipif_not_uses_gateway
 @pytest.mark.parametrize("machine, expected_response_code", DATA_201)

src/utilities/utilities.py

@@ -158,17 +158,19 @@ def list_directory():
     return common_fs_operation(request, "ls")
 
 
+
 ## parse ls output
-def ls_parse(request, retval):
+def ls_parse_folder(folder_content:str,path:str=""):
     # Example of ls output
-    # total 8
+    # total 3
     # lrwxrwxrwx 1 username groupname 46 2023-07-25T14:18:00 "filename" -> "target link"
     # -rw-rw-r-- 1 root root           0 2023-07-24T11:45:35 "root_file.txt"
-    # ...
-    file_pattern = (r'^(?P<type>\S)(?P<permissions>\S+)\s+\d+\s+(?P<user>\S+)\s+'
-                    r'(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<last_modified>(\d|-|T|:)+)\s+(?P<filename>.+)$')
-    matches = re.finditer(file_pattern, retval["msg"], re.MULTILINE)
+    # drwxrwxr-x 3 username groupname 4096 2023-07-24T11:45:35 "folder"
     file_list = []
+    file_pattern = (r'^(?P<type>\S)(?P<permissions>\S+)\s+\d+\s+(?P<user>\S+)\s+'
+                        r'(?P<group>\S+)\s+(?P<size>\d+)\s+(?P<last_modified>(\d|-|T|:)+)\s+(?P<filename>.+)$')
+    matches = re.finditer(file_pattern, folder_content, re.MULTILINE)
+
     for m in matches:
         tokens = shlex.split(m.group("filename"))
         if len(tokens) == 1:
@@ -184,7 +186,7 @@ def ls_parse(request, retval):
             continue
 
         file_list.append({
-            "name": name,
+            "name": path + name,
             "type": m.group("type"),
             "link_target": link_target,
             "user": m.group("user"),
@@ -193,7 +195,40 @@ def ls_parse(request, retval):
             "last_modified": m.group("last_modified"),
             "size": m.group("size")
         })
+    return file_list
+
+
+## parse ls output
+def ls_parse(request, retval):
+    # Example of ls output
+    # ".":
+    # total 8
+    # lrwxrwxrwx 1 username groupname 46 2023-07-25T14:18:00 "filename" -> "target link"
+    # -rw-rw-r-- 1 root root           0 2023-07-24T11:45:35 "root_file.txt"
+    # drwxrwxr-x 3 username groupname 4096 2023-07-24T11:45:35 "folder"
+    # "./folder":
+    # total 1
+    # -rw-rw-r-- 1 username groupname 0 2023-07-24T11:45:35 "file_in_folder.txt"
+    # ...
+
+    def remove_prefix(text, prefix):
+        return text[text.startswith(prefix) and len(prefix):]
 
+    file_list = []
+    #Check if ls has recursive folders
+    if(re.match(r'\"(.+)\":\n',retval["msg"])):
+        folders =  re.split(r'\"(.+)\":\n',retval["msg"])
+        root_folder = ""
+        for i in range(1,len(folders),2):
+            if i==1:
+                root_folder = folders[i]+"/"
+            folder_name = remove_prefix(folders[i]+"/",root_folder)
+            folder_content = folders[i+1]
+            file_list += ls_parse_folder(folder_content,folder_name)
+    else:
+        file_list += ls_parse_folder(retval["msg"])
+
+
     totalSize = len(file_list)
     logging.info(f"Length of file list: {len(file_list)}")
 
@@ -465,6 +500,9 @@ def common_fs_operation(request, command):
         if get_boolean_var(request.args.get("numericUid", False)):
             # do not resolve UID and GID to names
             options += "--numeric-uid-gid "
+        if get_boolean_var(request.args.get("recursive", False)):
+            # do not resolve UID and GID to names
+            options += "-R "
         action = f"ls -l --quoting-style=c {options} --time-style=+%Y-%m-%dT%H:%M:%S -- '{targetPath}'"
     elif command == "mkdir":
         try: