From d4b6853bf26c516c6cf33bbc42968221204ac7f6 Mon Sep 17 00:00:00 2001 From: Eirini Koutsaniti Date: Thu, 4 Apr 2024 07:53:50 +0000 Subject: [PATCH] Add /compress and /extract endpoints --- CHANGELOG.md | 4 +- deploy/test-build/cluster/Dockerfile | 2 + .../cluster/ssh/ssh_command_wrapper.sh | 2 +- deploy/test-build/cluster/test_zip.tar.gz | Bin 0 -> 374 bytes doc/openapi/firecrest-api.yaml | 312 +++++++++++++++++- doc/openapi/firecrest-developers-api.yaml | 296 +++++++++++++++++ src/common/cscs_api_common.py | 22 +- src/storage/storage.py | 38 ++- .../automated_tests/unit/test_unit_storage.py | 16 + src/utilities/utilities.py | 39 ++- 10 files changed, 714 insertions(+), 17 deletions(-) create mode 100644 deploy/test-build/cluster/test_zip.tar.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index accfe769..1534f79c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add the endpoints `/compute/nodes` and `/compute/nodes/{nodeName}` to retrieve information about nodes in the SLURM scheduling queue. +- Add the endpoints `/compute/nodes` and `/compute/nodes/{nodeName}` to retrieve information about nodes in the scheduling queue. +- Added endpoints `POST /utilities/compress`, `POST /utilities/extract`, `POST /storage/xfer-internal/compress` and `POST /storage/xfer-internal/extract` for file compression and extraction. + ### Changed ### Fixed diff --git a/deploy/test-build/cluster/Dockerfile b/deploy/test-build/cluster/Dockerfile index ed315008..200378f5 100644 --- a/deploy/test-build/cluster/Dockerfile +++ b/deploy/test-build/cluster/Dockerfile @@ -117,10 +117,12 @@ ADD cluster/test_sbatch.sh /srv/f7t/. ADD cluster/test_sbatch.sh /srv/f7t/test_sbatch_forbidden.sh ADD cluster/test_sbatch.sh /srv/f7t/test_sbatch_rm.sh ADD cluster/test_sbatch.sh /srv/f7t/test_sbatch_mv.sh +COPY cluster/test_zip.tar.gz /srv/f7t/test_zip.tar.gz RUN chmod 777 /srv/f7t RUN chmod 555 /srv/f7t/test_sbatch.sh RUN chmod 777 /srv/f7t/test_sbatch_rm.sh RUN chmod 777 /srv/f7t/test_sbatch_mv.sh +RUN chmod 777 /srv/f7t/test_zip.tar.gz RUN chmod 700 /srv/f7t/test_sbatch_forbidden.sh ENTRYPOINT ["supervisord"] diff --git a/deploy/test-build/cluster/ssh/ssh_command_wrapper.sh b/deploy/test-build/cluster/ssh/ssh_command_wrapper.sh index 7da5d231..59601cb7 100644 --- a/deploy/test-build/cluster/ssh/ssh_command_wrapper.sh +++ b/deploy/test-build/cluster/ssh/ssh_command_wrapper.sh @@ -35,7 +35,7 @@ case "$command" in tmp2=${tmp1#* } command2=${tmp2%% *} # remove options case "$command2" in - base64|cat|chmod|chown|cp|curl|file|head|id|ln|ls|mkdir|mv|rm|sacct|sbatch|scancel|scontrol|sha256sum|squeue|stat|tail|touch) + base64|cat|chmod|chown|cp|curl|file|head|id|ln|ls|mkdir|mv|rm|sacct|sbatch|scancel|scontrol|sha256sum|squeue|stat|tail|touch|tar|unzip) ;; rsvmgmt) # advance reservation command diff --git a/deploy/test-build/cluster/test_zip.tar.gz b/deploy/test-build/cluster/test_zip.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..31e2d622a1598eb12db5829730fb6c6d0ffd4592 GIT binary patch literal 374 zcmV-+0g3(}iwFS5v*~331MSmIZ-Ouw2XM~)6g7B7T6s&Bc-YL6opn1Kgc`db5L#Wb zPrr5ygUgm37Tk;f-vMm&vdj`I&CO*lxPkz6JDBS)xOl;Mm8dA%)jHQbXON&uK!WG>p!2U?Lfa_8yv^s zr~Qvu(CdF>`)>vgWdoh(#D(|Y9rxiraLh^KIzI# zt?UcatJ2a`aCXI5Z?dd%p=O*AQYh+|mA{dNAiFBx7lIbjY#_32MIV;8i_xh4JhlTu zR3Su7W3wA^srmAFIbYaK+oJj@H}&S)4&T$+OdlHxSpXH}p_$|l$vtO2fj{Khk@{jz#=70KI@*lVP50U?KK>jZ~{{R3000000 U000000Pw%O0MX&YH2^3807K5YIRF3v literal 0 HcmV?d00001 diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index be974463..e4c1d6cf 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -1323,6 +1323,161 @@ paths: description: Command has finished with timeout signal schema: type: string + '/utilities/compress': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: Compress file/directory + description: >- + Compress files using gzip compression. You can name the output file as you like, but typically these files have a .tar.gz extension. + tags: + - Utilities + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: Absolute filesystem path to file to be compressed + targetPath: + type: string + description: Absolute filesystem path of the compressed file + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/directory + targetPath: /home/user/file.tar.gz + responses: + '201': + description: Object compress succesful + content: + application/json: + schema: + $ref: '#/components/schemas/Utilities-ok' + '400': + description: Error in compress operation + content: + text/plain: + schema: + $ref: '#/components/schemas/Utilities-notok' + headers: + X-Machine-Does-Not-Exist: + description: Machine does not exist + schema: + type: string + X-Machine-Not-Available: + description: Machine is not available + schema: + type: string + X-Permission-Denied: + description: User does not have permissions to access machine or paths + schema: + type: string + X-Invalid-Path: + description: is an invalid path + schema: + type: string + X-Timeout: + description: Command has finished with timeout signal + schema: + type: string + X-Exists: + description: already exists + schema: + type: string + X-Not-Found: + description: not found + schema: + type: string + '/utilities/extract': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: Extract file + description: >- + Extract files. If you don't select the `extension`, FirecREST will try to guess the right command based on the extension of the sourcePath. Supported extensions are `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`. + tags: + - Utilities + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: Absolute filesystem path to file to be extracted + targetPath: + type: string + description: Absolute filesystem path where the {sourcePath} is extracted + extension: + type: string + description: Possible values are `auto`, `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2` + default: "auto" + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/file.tar.gz + targetPath: /home/user/directory + extension: .tar.gz + responses: + '201': + description: Object extract succesful + content: + application/json: + schema: + $ref: '#/components/schemas/Utilities-ok' + '400': + description: Error in extract operation + content: + text/plain: + schema: + $ref: '#/components/schemas/Utilities-notok' + headers: + X-Machine-Does-Not-Exist: + description: Machine does not exist + schema: + type: string + X-Machine-Not-Available: + description: Machine is not available + schema: + type: string + X-Permission-Denied: + description: User does not have permissions to access machine or paths + schema: + type: string + X-Invalid-Path: + description: is an invalid path + schema: + type: string + X-Timeout: + description: Command has finished with timeout signal + schema: + type: string + X-Exists: + description: already exists + schema: + type: string + X-Not-Found: + description: not found + schema: + type: string '/compute/jobs/upload': parameters: - in: header @@ -2035,6 +2190,160 @@ paths: application/json: schema: $ref: '#/components/schemas/Task-Creation-Error' + '/storage/xfer-internal/compress': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: compress files + description: >- + 'Compress files using gzip compression. You can name the output file + as you like, but typically these files have a .tar.gz or .tgz extension. + Possible to stage-out jobs providing the workload manager Id of a + production job. Reference: + https://user.cscs.ch/storage/data_transfer/internal_transfer/' + tags: + - Storage + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: source path to the location filesystem + targetPath: + type: string + description: Absolute path to destination + jobName: + type: string + description: job name the copy operation + default: cp-job + time: + type: string + description: >- + 'Limit on the total run time of the copy. Acceptable time formats + \'minutes\', \'minutes:seconds\', \'hours:minutes:seconds\', + \'days-hours\', \'days-hours:minutes\' and + \'days-hours:minutes:seconds\'. Note: for stage-in queue a + xfer job.' + default: '02:00:00' + stageOutJobId: + type: string + description: Copy data after job with id {stageOutJobId} is completed + default: null + account: + type: string + description: Name of the account associated to the user in the scheduler. If not set, system default is taken. + default: null + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/origin + targetPath: /home/user/destination + jobName: cp-firecrest-job + stageOutJobId: "123456" + time: "2-03:00:00" + responses: + '201': + description: Task created + content: + application/json: + schema: + $ref: '#/components/schemas/Task-Creation-Success' + '400': + description: Task creation error + content: + application/json: + schema: + $ref: '#/components/schemas/Task-Creation-Error' + '/storage/xfer-internal/extract': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: extract files + description: >- + 'Extract files. If you don't select the `extension`, FirecREST will + try to guess the right command based on the extension of the sourcePath. + Supported extensions are `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`. + Possible to stage-out jobs providing the workload manager Id of a + production job. Reference: + https://user.cscs.ch/storage/data_transfer/internal_transfer/' + tags: + - Storage + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: source path to the location filesystem + targetPath: + type: string + description: Absolute path to destination + jobName: + type: string + description: job name the copy operation + default: cp-job + time: + type: string + description: >- + 'Limit on the total run time of the copy. Acceptable time formats + \'minutes\', \'minutes:seconds\', \'hours:minutes:seconds\', + \'days-hours\', \'days-hours:minutes\' and + \'days-hours:minutes:seconds\'. Note: for stage-in queue a + xfer job.' + default: '02:00:00' + stageOutJobId: + type: string + description: Copy data after job with id {stageOutJobId} is completed + default: null + account: + type: string + description: Name of the account associated to the user in the scheduler. If not set, system default is taken. + default: null + extension: + type: string + description: Possible values are `auto`, `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2` + default: "auto" + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/origin + targetPath: /home/user/destination + jobName: cp-firecrest-job + stageOutJobId: "123456" + time: "2-03:00:00" + extension: ".tar.gz" + responses: + '201': + description: Task created + content: + application/json: + schema: + $ref: '#/components/schemas/Task-Creation-Success' + '400': + description: Task creation error + content: + application/json: + schema: + $ref: '#/components/schemas/Task-Creation-Error' '/storage/xfer-external/upload': parameters: - in: header @@ -2829,7 +3138,6 @@ components: enum: - 200 - 400 - Filesystems: type: object properties: @@ -2840,8 +3148,6 @@ components: type: array items: $ref: '#/components/schemas/Filesystem' - - Job: type: object required: diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index ce9a26e0..fd1a0388 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -1311,6 +1311,161 @@ paths: description: Command has finished with timeout signal schema: type: string + '/utilities/compress': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: Compress file/directory + description: >- + Compress files using gzip compression. You can name the output file as you like, but typically these files have a .tar.gz extension. + tags: + - Utilities + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: Absolute filesystem path to file to be compressed + targetPath: + type: string + description: Absolute filesystem path of the compressed file + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/directory + targetPath: /home/user/file.tar.gz + responses: + '201': + description: Object compress succesful + content: + application/json: + schema: + $ref: '#/components/schemas/Utilities-ok' + '400': + description: Error in compress operation + content: + text/plain: + schema: + $ref: '#/components/schemas/Utilities-notok' + headers: + X-Machine-Does-Not-Exist: + description: Machine does not exist + schema: + type: string + X-Machine-Not-Available: + description: Machine is not available + schema: + type: string + X-Permission-Denied: + description: User does not have permissions to access machine or paths + schema: + type: string + X-Invalid-Path: + description: is an invalid path + schema: + type: string + X-Timeout: + description: Command has finished with timeout signal + schema: + type: string + X-Exists: + description: already exists + schema: + type: string + X-Not-Found: + description: not found + schema: + type: string + '/utilities/extract': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: Extract file + description: >- + Extract files. If you don't select the `extension`, FirecREST will try to guess the right command based on the extension of the sourcePath. Supported extensions are `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`. + tags: + - Utilities + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: Absolute filesystem path to file to be extracted + targetPath: + type: string + description: Absolute filesystem path where the {sourcePath} is extracted + extension: + type: string + description: Possible values are `auto`, `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2` + default: "auto" + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/file.tar.gz + targetPath: /home/user/directory + extension: .tar.gz + responses: + '201': + description: Object extract succesful + content: + application/json: + schema: + $ref: '#/components/schemas/Utilities-ok' + '400': + description: Error in extract operation + content: + text/plain: + schema: + $ref: '#/components/schemas/Utilities-notok' + headers: + X-Machine-Does-Not-Exist: + description: Machine does not exist + schema: + type: string + X-Machine-Not-Available: + description: Machine is not available + schema: + type: string + X-Permission-Denied: + description: User does not have permissions to access machine or paths + schema: + type: string + X-Invalid-Path: + description: is an invalid path + schema: + type: string + X-Timeout: + description: Command has finished with timeout signal + schema: + type: string + X-Exists: + description: already exists + schema: + type: string + X-Not-Found: + description: not found + schema: + type: string '/compute/jobs/upload': parameters: - in: header @@ -2023,6 +2178,147 @@ paths: application/json: schema: $ref: '#/components/schemas/Task-Creation-Error' + '/storage/xfer-internal/compress': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: compress files + description: >- + 'Compress files using gzip compression. You can name the output file + as you like, but typically these files have a .tar.gz or .tgz extension. + Possible to stage-out jobs providing the workload manager Id of a + production job. Reference: + https://user.cscs.ch/storage/data_transfer/internal_transfer/' + tags: + - Storage + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: source path to the location filesystem + targetPath: + type: string + description: Absolute path to destination + jobName: + type: string + description: job name the copy operation + default: cp-job + time: + type: string + description: >- + 'Limit on the total run time of the copy. Acceptable time formats + \'minutes\', \'minutes:seconds\', \'hours:minutes:seconds\', + \'days-hours\', \'days-hours:minutes\' and + \'days-hours:minutes:seconds\'. Note: for stage-in queue a + xfer job.' + default: '02:00:00' + stageOutJobId: + type: string + description: Copy data after job with id {stageOutJobId} is completed + default: null + account: + type: string + description: Name of the account associated to the user in the scheduler. If not set, system default is taken. + default: null + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/origin + targetPath: /home/user/destination + jobName: cp-firecrest-job + stageOutJobId: "123456" + time: "2-03:00:00" + responses: + '201': + description: Task created + content: + application/json: + schema: + $ref: '#/components/schemas/Task-Creation-Success' + '400': + description: Task creation error + content: + application/json: + schema: + $ref: '#/components/schemas/Task-Creation-Error' + '/storage/xfer-internal/extract': + parameters: + - in: header + name: X-Machine-Name + description: The system name + required: true + schema: + type: string + post: + summary: extract files + description: >- + 'Extract files. If you don't select the `extension`, FirecREST will + try to guess the right command based on the extension of the sourcePath. + Supported extensions are `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2`. + Possible to stage-out jobs providing the workload manager Id of a + production job. Reference: + https://user.cscs.ch/storage/data_transfer/internal_transfer/' + tags: + - Storage + requestBody: + required: true + content: + 'multipart/form-data': + schema: + type: object + properties: + sourcePath: + type: string + description: source path to the location filesystem + targetPath: + type: string + description: Absolute path to destination + jobName: + type: string + description: job name the copy operation + default: cp-job + time: + type: string + description: >- + 'Limit on the total run time of the copy. Acceptable time formats + \'minutes\', \'minutes:seconds\', \'hours:minutes:seconds\', + \'days-hours\', \'days-hours:minutes\' and + \'days-hours:minutes:seconds\'. Note: for stage-in queue a + xfer job.' + default: '02:00:00' + stageOutJobId: + type: string + description: Copy data after job with id {stageOutJobId} is completed + default: null + account: + type: string + description: Name of the account associated to the user in the scheduler. If not set, system default is taken. + default: null + extension: + type: string + description: Possible values are `auto`, `.zip`, `.tar`, `.tgz`, `.gz` and `.bz2` + default: "auto" + required: + - sourcePath + - targetPath + example: + sourcePath: /home/user/origin + targetPath: /home/user/destination + jobName: cp-firecrest-job + stageOutJobId: "123456" + time: "2-03:00:00" + extension: ".tar.gz" '/storage/xfer-external/upload': parameters: - in: header diff --git a/src/common/cscs_api_common.py b/src/common/cscs_api_common.py index 28168b95..d97b72a4 100644 --- a/src/common/cscs_api_common.py +++ b/src/common/cscs_api_common.py @@ -260,7 +260,7 @@ def in_str(stringval, substring): return substring in stringval else: return False - + # SSH certificates creation # returns pub key certificate name @@ -1059,3 +1059,23 @@ def setup_logging(logging, service): logging.info("DEBUG_MODE: False") return logger + +def extract_command(file_path, output_directory, type="auto"): + '''Return the appropriate command based on the file extension, or None if not supported + ''' + # Map extension to command + command_map = { + ".zip": f"unzip -o '{file_path}' -d '{output_directory}'", + ".tar": f"tar -xf '{file_path}' -C '{output_directory}'", + ".bz2": f"tar -xjf '{file_path}' -C '{output_directory}'", + ".gz": f"tar -xzf '{file_path}' -C '{output_directory}'", + ".tgz": f"tar -xzf '{file_path}' -C '{output_directory}'", + } + + if type == "auto": + # Extract the file extension + _, extension = os.path.splitext(file_path) + else: + extension = type + + return command_map.get(extension, None) diff --git a/src/storage/storage.py b/src/storage/storage.py index 3df12c48..5f6a0afc 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -19,6 +19,7 @@ from cscs_api_common import create_certificate from cscs_api_common import in_str from cscs_api_common import is_valid_file, is_valid_dir, check_command_error, get_boolean_var, get_null_var, validate_input, setup_logging +from cscs_api_common import extract_command from schedulers import JobScript @@ -945,6 +946,20 @@ def internal_rm(): return internal_operation(request, "rm") +# Internal compression transfer via the scheduler xfer partition: +@app.route("/xfer-internal/compress", methods=["POST"]) +@check_auth_header +def internal_compress(): + return internal_operation(request, "compress") + + +# Internal extraction transfer via the scheduler xfer partition: +@app.route("/xfer-internal/extract", methods=["POST"]) +@check_auth_header +def internal_extract(): + return internal_operation(request, "extract") + + # common code for internal cp, mv, rsync, rm def internal_operation(request, command): @@ -973,7 +988,7 @@ def internal_operation(request, command): [headers, ID] = get_tracing_headers(request) # using actual_command to add options to check sanity of the command to be executed actual_command = "" - if command in ['cp', 'mv', 'rsync']: + if command in ['cp', 'mv', 'rsync', 'compress', 'extract']: sourcePath = request.form.get("sourcePath", None) # path to get file in cluster v = validate_input(sourcePath) if v != "": @@ -988,20 +1003,27 @@ def internal_operation(request, command): app.logger.info(f"_targetPath={_targetPath}") if command == "cp": - actual_command = "cp --force -dR --preserve=all -- " + actual_command = f"cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'" elif command == "mv": - actual_command = "mv --force -- " + actual_command = f"mv --force -- '{sourcePath}' '{targetPath}'" + elif command == "rsync": + actual_command = f"rsync -av -- '{sourcePath}' '{targetPath}'" + elif command == "compress": + basedir = os.path.dirname(sourcePath) + file_path = os.path.basename(sourcePath) + actual_command = f"tar -czf '{targetPath}' -C '{basedir}' '{file_path}'" else: - actual_command = "rsync -av -- " + extraction_type = request.form.get("type", "auto") + actual_command = extract_command(sourcePath, targetPath, type=extraction_type) + if not actual_command: + return jsonify(description=f"Error on {command} operation", error=f"Unsupported file format in {sourcePath}."), 400 + elif command == "rm": sourcePath = "" - actual_command = "rm -rf -- " + actual_command = f"rm -rf -- '{targetPath}'" else: return jsonify(error=f"Command {command} not allowed"), 400 - # don't add tracing ID, we'll be executed by srun - actual_command = f"{actual_command} '{sourcePath}' '{targetPath}'" - jobName = request.form.get("jobName", "") if jobName == "": jobName = command + "-job" diff --git a/src/tests/automated_tests/unit/test_unit_storage.py b/src/tests/automated_tests/unit/test_unit_storage.py index 9f7be6d3..35dce958 100644 --- a/src/tests/automated_tests/unit/test_unit_storage.py +++ b/src/tests/automated_tests/unit/test_unit_storage.py @@ -85,6 +85,22 @@ def test_internal_rsync(headers): resp = requests.post(url, headers=headers,data=data, verify= (f"{SSL_PATH}{SSL_CRT}" if USE_SSL else False)) assert resp.status_code == 201 +@skipif_not_uses_gateway +def test_internal_compress(headers): + headers["X-Machine-Name"] = machine + data = {"sourcePath": "/srv/f7t", "targetPath": USER_HOME + "/f7t.tar.gz", "account": "test"} + url = f"{STORAGE_URL}/xfer-internal/compress" + resp = requests.post(url, headers=headers,data=data, verify= (f"{SSL_PATH}{SSL_CRT}" if USE_SSL else False)) + assert resp.status_code == 201 + +@skipif_not_uses_gateway +def test_internal_compress(headers): + headers["X-Machine-Name"] = machine + data = {"sourcePath": "/srv/f7t/test_zip.tar.gz", "targetPath": USER_HOME, "account": "test", "extension": "tar.gz"} + url = f"{STORAGE_URL}/xfer-internal/extract" + resp = requests.post(url, headers=headers,data=data, verify= (f"{SSL_PATH}{SSL_CRT}" if USE_SSL else False)) + assert resp.status_code == 201 + @skipif_not_uses_gateway def test_internal_rm(headers): headers["X-Machine-Name"] = machine diff --git a/src/utilities/utilities.py b/src/utilities/utilities.py index 4d8d08ae..739c1413 100644 --- a/src/utilities/utilities.py +++ b/src/utilities/utilities.py @@ -19,7 +19,7 @@ import re import shlex -from cscs_api_common import check_auth_header, exec_remote_command, check_command_error, get_boolean_var, validate_input, setup_logging +from cscs_api_common import check_auth_header, exec_remote_command, check_command_error, get_boolean_var, validate_input, setup_logging, extract_command CERTIFICATOR_URL = os.environ.get("F7T_CERTIFICATOR_URL") UTILITIES_PORT = os.environ.get("F7T_UTILITIES_PORT", 5000) @@ -314,6 +314,27 @@ def rename(): def copy(): return common_fs_operation(request, "copy") +## compress with tar +## params: +## - sourcepath: Filesystem path that will be compressed (Str) *required +## - targetpath: Filesystem path of copied object (Str) *required +## - machinename: str *required +@app.route("/compress", methods=["POST"]) +@check_auth_header +def compress(): + return common_fs_operation(request, "compress") + +## extract files +## params: +## - sourcepath: Filesystem path of object to be decompressed (Str) *required +## - targetpath: Filesystem path of extracted files (Str) *required +## - machinename: str *required +## - type: Extension of the file (Str) +@app.route("/extract", methods=["POST"]) +@check_auth_header +def extract(): + return common_fs_operation(request, "extract") + ## common code for file operations: def common_fs_operation(request, command): @@ -347,7 +368,7 @@ def common_fs_operation(request, command): if v != "" and command != "whoami": return jsonify(description=f"Error on {command} operation", error=f"'{tn}' {v}"), 400 - if command in ['copy', 'rename']: + if command in ['copy', 'rename', 'compress', 'extract']: sourcePath = request.form.get("sourcePath", None) v = validate_input(sourcePath) if v != "": @@ -380,6 +401,18 @@ def common_fs_operation(request, command): elif command == "copy": # -r is for recursivelly copy files into directories action = f"cp --force -dR --preserve=all -- '{sourcePath}' '{targetPath}'" + success_code = 201 + elif command == "compress": + basedir = os.path.dirname(sourcePath) + file_path = os.path.basename(sourcePath) + action = f"tar -czvf '{targetPath}' -C '{basedir}' '{file_path}'" + success_code = 201 + elif command == "extract": + extraction_type = request.form.get("type", "auto") + action = extract_command(sourcePath, targetPath, type=extraction_type) + if not action: + return jsonify(description=f"Error on {command} operation", error=f"Unsupported file format in {sourcePath}."), 400 + success_code = 201 elif command == "file": # -b: do not prepend filenames to output lines @@ -556,7 +589,7 @@ def common_fs_operation(request, command): groups = [] group_list = whoami_response[whoami_response.find("=",gname_j)+1:].split(",") - + for group in group_list: gname_i = group.find("(", 0) gname_j = group.find(")", 0)