From 11cf28bf0489bb45bbae3c69ca4948f4faec33bb Mon Sep 17 00:00:00 2001 From: Elia Palme Date: Fri, 5 Apr 2024 11:10:18 +0200 Subject: [PATCH 1/6] Implementation of ls recursive --- src/utilities/utilities.py | 50 ++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/src/utilities/utilities.py b/src/utilities/utilities.py index 739c1413..aa30e58d 100644 --- a/src/utilities/utilities.py +++ b/src/utilities/utilities.py @@ -158,17 +158,19 @@ def list_directory(): return common_fs_operation(request, "ls") + ## parse ls output -def ls_parse(request, retval): +def ls_parse_folder(folder_content:str,path:str=""): # Example of ls output - # total 8 + # total 3 # lrwxrwxrwx 1 username groupname 46 2023-07-25T14:18:00 "filename" -> "target link" # -rw-rw-r-- 1 root root 0 2023-07-24T11:45:35 "root_file.txt" - # ... - file_pattern = (r'^(?P\S)(?P\S+)\s+\d+\s+(?P\S+)\s+' - r'(?P\S+)\s+(?P\d+)\s+(?P(\d|-|T|:)+)\s+(?P.+)$') - matches = re.finditer(file_pattern, retval["msg"], re.MULTILINE) + # drwxrwxr-x 3 username groupname 4096 2023-07-24T11:45:35 "folder" file_list = [] + file_pattern = (r'^(?P\S)(?P\S+)\s+\d+\s+(?P\S+)\s+' + r'(?P\S+)\s+(?P\d+)\s+(?P(\d|-|T|:)+)\s+(?P.+)$') + matches = re.finditer(file_pattern, folder_content, re.MULTILINE) + for m in matches: tokens = shlex.split(m.group("filename")) if len(tokens) == 1: @@ -184,7 +186,7 @@ def ls_parse(request, retval): continue file_list.append({ - "name": name, + "name": path + name, "type": m.group("type"), "link_target": link_target, "user": m.group("user"), @@ -193,7 +195,38 @@ def ls_parse(request, retval): "last_modified": m.group("last_modified"), "size": m.group("size") }) + return file_list + + +## parse ls output +def ls_parse(request, retval): + # Example of ls output + # ".": + # total 8 + # lrwxrwxrwx 1 username groupname 46 2023-07-25T14:18:00 "filename" -> "target link" + # -rw-rw-r-- 1 root root 0 2023-07-24T11:45:35 "root_file.txt" + # drwxrwxr-x 3 username groupname 4096 2023-07-24T11:45:35 "folder" + # "./folder": + # total 1 + # -rw-rw-r-- 1 username groupname 0 2023-07-24T11:45:35 "file_in_folder.txt" + # ... + file_list = [] + #Check if ls has recursive folders + if(re.match(r'\"(.+)\":\n',retval["msg"])): + folders = re.split(r'\"(.+)\":\n',retval["msg"]) + root_folder = "" + for i in range(1,len(folders),2): + if i==1: + root_folder = folders[i]+"/" + + folder_name = (folders[i]+"/").replace(root_folder,"") + folder_content = folders[i+1] + file_list += ls_parse_folder(folder_content,folder_name) + else: + file_list += ls_parse_folder(retval["msg"]) + + totalSize = len(file_list) logging.info(f"Length of file list: {len(file_list)}") @@ -465,6 +498,9 @@ def common_fs_operation(request, command): if get_boolean_var(request.args.get("numericUid", False)): # do not resolve UID and GID to names options += "--numeric-uid-gid " + if get_boolean_var(request.args.get("recursive", False)): + # do not resolve UID and GID to names + options += "-R" action = f"ls -l --quoting-style=c {options} --time-style=+%Y-%m-%dT%H:%M:%S -- '{targetPath}'" elif command == "mkdir": try: From e29bd474b77cfa8122ab963b60bc1d8109c1e7d6 Mon Sep 17 00:00:00 2001 From: Elia Palme Date: Fri, 5 Apr 2024 11:10:34 +0200 Subject: [PATCH 2/6] Tests for ls recursive --- src/tests/automated_tests/unit/test_unit_utilities.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/tests/automated_tests/unit/test_unit_utilities.py b/src/tests/automated_tests/unit/test_unit_utilities.py index 36b1f996..a803eb95 100644 --- a/src/tests/automated_tests/unit/test_unit_utilities.py +++ b/src/tests/automated_tests/unit/test_unit_utilities.py @@ -245,6 +245,17 @@ def test_list_directory(machine, targetPath, expected_response_code, headers): print(resp.headers) assert resp.status_code == expected_response_code +@skipif_not_uses_gateway +@pytest.mark.parametrize("machine, targetPath, expected_response_code", DATA_LS) +def test_list_directory_recursive(machine, targetPath, expected_response_code, headers): + params = {"targetPath": targetPath, "recursive" : "true"} + url = f"{UTILITIES_URL}/ls" + headers.update({"X-Machine-Name": machine}) + resp = requests.get(url, headers=headers, params=params, verify= (f"{SSL_PATH}{SSL_CRT}" if USE_SSL else False)) + print(json.dumps(resp.json(),indent=2)) + print(resp.headers) + assert resp.status_code == expected_response_code + @skipif_not_uses_gateway @pytest.mark.parametrize("machine, expected_response_code", DATA_201) From 474fb59b2ac00c6ed993ec4809762417c1302716 Mon Sep 17 00:00:00 2001 From: Elia Palme Date: Fri, 5 Apr 2024 11:14:56 +0200 Subject: [PATCH 3/6] fix ls "-R" option --- src/utilities/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utilities/utilities.py b/src/utilities/utilities.py index aa30e58d..ac7e1fb9 100644 --- a/src/utilities/utilities.py +++ b/src/utilities/utilities.py @@ -500,7 +500,7 @@ def common_fs_operation(request, command): options += "--numeric-uid-gid " if get_boolean_var(request.args.get("recursive", False)): # do not resolve UID and GID to names - options += "-R" + options += "-R " action = f"ls -l --quoting-style=c {options} --time-style=+%Y-%m-%dT%H:%M:%S -- '{targetPath}'" elif command == "mkdir": try: From eb2a889c00aec8be272c09840d3657b8ba6ea45c Mon Sep 17 00:00:00 2001 From: Elia Palme Date: Fri, 5 Apr 2024 11:17:26 +0200 Subject: [PATCH 4/6] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1534f79c..d8c3e122 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add the endpoints `/compute/nodes` and `/compute/nodes/{nodeName}` to retrieve information about nodes in the scheduling queue. - Added endpoints `POST /utilities/compress`, `POST /utilities/extract`, `POST /storage/xfer-internal/compress` and `POST /storage/xfer-internal/extract` for file compression and extraction. +- Added recurisive option to ls utilities command `&recursive=true` ### Changed From 9671a5b268c138ee2692ca6f437a0f3ee42beefe Mon Sep 17 00:00:00 2001 From: Elia Palme Date: Fri, 5 Apr 2024 15:23:53 +0200 Subject: [PATCH 5/6] using proper remove_prefix for path clean-up --- src/utilities/utilities.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/utilities/utilities.py b/src/utilities/utilities.py index ac7e1fb9..b412d9f6 100644 --- a/src/utilities/utilities.py +++ b/src/utilities/utilities.py @@ -210,8 +210,11 @@ def ls_parse(request, retval): # total 1 # -rw-rw-r-- 1 username groupname 0 2023-07-24T11:45:35 "file_in_folder.txt" # ... - file_list = [] + + def remove_prefix(text, prefix): + return text[text.startswith(prefix) and len(prefix):] + file_list = [] #Check if ls has recursive folders if(re.match(r'\"(.+)\":\n',retval["msg"])): folders = re.split(r'\"(.+)\":\n',retval["msg"]) @@ -219,8 +222,7 @@ def ls_parse(request, retval): for i in range(1,len(folders),2): if i==1: root_folder = folders[i]+"/" - - folder_name = (folders[i]+"/").replace(root_folder,"") + folder_name = remove_prefix(folders[i]+"/",root_folder) folder_content = folders[i+1] file_list += ls_parse_folder(folder_content,folder_name) else: From de4c4447261d37994ca80572f456d58199e5d70b Mon Sep 17 00:00:00 2001 From: Elia Palme Date: Fri, 5 Apr 2024 15:27:29 +0200 Subject: [PATCH 6/6] Updated openapi doc --- doc/openapi/firecrest-api.yaml | 6 ++++++ doc/openapi/firecrest-developers-api.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index e4c1d6cf..c0d62647 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -207,6 +207,12 @@ paths: schema: type: boolean default: false + - name: recursive + in: query + description: List files recursively + schema: + type: boolean + default: false # - $ref: '#/components/parameters/pageSize' # - $ref: '#/components/parameters/pageNumber' responses: diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index fd1a0388..3da49fa2 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -195,6 +195,12 @@ paths: schema: type: boolean default: false + - name: recursive + in: query + description: List files recursively + schema: + type: boolean + default: false # - $ref: '#/components/parameters/pageSize' # - $ref: '#/components/parameters/pageNumber' responses: