From c0762210dd903eff5adb4bf20b615aae4f555032 Mon Sep 17 00:00:00 2001
From: MalinAhlberg <ahlberg.malin@gmail.com>
Date: Tue, 17 Dec 2024 14:13:38 +0100
Subject: [PATCH 1/5] [tests] split integration tests into separate files

---
 .../integration/tests/10_encrypt_decrypt.sh   | 135 +++++++
 .github/integration/tests/20_upload.sh        | 115 ++++++
 .github/integration/tests/30_list.sh          |  33 ++
 .github/integration/tests/40_download.sh      | 128 ++++++
 .github/integration/tests/tests.sh            | 373 ------------------
 5 files changed, 411 insertions(+), 373 deletions(-)
 create mode 100755 .github/integration/tests/10_encrypt_decrypt.sh
 create mode 100755 .github/integration/tests/20_upload.sh
 create mode 100755 .github/integration/tests/30_list.sh
 create mode 100755 .github/integration/tests/40_download.sh
 delete mode 100755 .github/integration/tests/tests.sh

diff --git a/.github/integration/tests/10_encrypt_decrypt.sh b/.github/integration/tests/10_encrypt_decrypt.sh
new file mode 100755
index 00000000..49af6d87
--- /dev/null
+++ b/.github/integration/tests/10_encrypt_decrypt.sh
@@ -0,0 +1,135 @@
+#!/bin/bash
+set -e
+
+# Function checking that a file was encrypted
+function check_encypted_file() {
+
+    for k in $1
+    do
+        output=$(head -c 8 "$k")
+
+        if [[ "$output" = "crypt4gh"  ]]; then
+            echo "Encrypted data file"
+        else
+            echo "Failed to encrypt file"
+            exit 1
+        fi
+    done
+}
+
+# inferred from access_key in testing/s3cmd.conf
+user=test_dummy.org
+
+# Create random file
+dd if=/dev/urandom of=data_file count=1 bs=1M
+
+# Create key pair
+if ( yes "" | ./sda-cli createKey sda_key ) ; then
+    echo "Created key pair for encryption"
+else
+    echo "Failed to create key pair for encryption"
+    exit 1
+fi
+
+# Encrypt a file
+./sda-cli encrypt -key sda_key.pub.pem data_file
+
+files="data_file.c4gh"
+check_encypted_file $files
+
+
+# Create and encrypt multiple files in a folder
+
+# Create folder and encrypt files in it
+cp data_file data_file1
+mkdir data_files_enc
+./sda-cli encrypt -key sda_key.pub.pem -outdir data_files_enc data_file data_file1
+
+check_encypted_file "data_files_enc/data_file.c4gh data_files_enc/data_file1.c4gh"
+
+
+# Create folder with subfolder structure and add some encrypted files
+mkdir data_files_enc/dir1 data_files_enc/dir1/dir2
+cp data_files_enc/data_file.c4gh data_files_enc/data_file3.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/data_file.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file2.c4gh
+
+# Test multiple pub key encryption
+
+# Create another couple of key-pairs
+for c in 1 2
+do
+    if ( yes "" | ./sda-cli createKey sda_key$c ) ; then
+        echo "Created key pair for encryption"
+    else
+        echo "Failed to create key pair for encryption"
+        exit 1
+fi
+done
+
+# Create file with concatenated pub keys
+cat sda_key1.pub.pem sda_key2.pub.pem > sda_keys
+
+
+# Create test files
+cp data_file data_file_keys
+
+# Encrypt with multiple key flag calls
+./sda-cli encrypt -key sda_key.pub.pem -key sda_key2.pub.pem data_file_keys
+check_encypted_file "data_file_keys.c4gh"
+# Decrypt file with both keys
+for key in sda_key sda_key2
+do
+    rm data_file_keys
+    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
+    if [ -f data_file_keys ]; then
+        echo "Decrypted data file"
+    else
+        echo "Failed to decrypt data file with $key"
+        exit 1
+    fi
+done
+echo "Could decrypt with both keys from multiple key flag"
+rm data_file_keys.c4gh
+
+
+# Encrypt with concatenated key file and a key flag call
+./sda-cli encrypt -key sda_key.pub.pem -key sda_keys data_file_keys
+check_encypted_file "data_file_keys.c4gh"
+
+# Decrypt file with both keys
+for key in sda_key sda_key1 sda_key2
+do
+    rm data_file_keys
+    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
+    if [ -f data_file_keys ]; then
+        echo "Decrypted data file"
+    else
+        echo "Failed to decrypt data file with $key"
+        exit 1
+    fi
+done
+echo "Could decrypt with both keys from concatenated key"
+rm data_file_keys.c4gh
+
+# Encrypt with concatenated key file
+./sda-cli encrypt -key sda_keys data_file_keys
+check_encypted_file "data_file_keys.c4gh"
+
+# Decrypt file with all keys
+for key in sda_key1 sda_key2
+do
+    rm data_file_keys
+    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
+    if [ -f data_file_keys ]; then
+        echo "Decrypted data file"
+    else
+        echo "Failed to decrypt data file with $key"
+        exit 1
+    fi
+done
+
+echo "Could decrypt with all keys from concatenated key"
+
+echo "Integration tests for sda-cli encrypt finished successfully"
\ No newline at end of file
diff --git a/.github/integration/tests/20_upload.sh b/.github/integration/tests/20_upload.sh
new file mode 100755
index 00000000..dd6f0994
--- /dev/null
+++ b/.github/integration/tests/20_upload.sh
@@ -0,0 +1,115 @@
+#!/bin/bash
+set -e
+# inferred from access_key in testing/s3cmd.conf
+user=test_dummy.org
+
+
+# Function checking that a file was encrypted
+function check_encypted_file() {
+
+    for k in $1
+    do
+        output=$(head -c 8 "$k")
+
+        if [[ "$output" = "crypt4gh"  ]]; then
+            echo "Encrypted data file"
+        else
+            echo "Failed to encrypt file"
+            exit 1
+        fi
+    done
+}
+
+# Function checking that a file was uploaded to the S3 backend
+function check_uploaded_file() {
+    # TODO if emtpy, this will fail silently
+    output=$(s3cmd -c testing/directS3 ls s3://"$1" | grep -q "$2")
+    if $output ; then
+        echo "Uploaded encrypted file to s3 backend"
+    else
+        echo "Failed to upload file to s3 backend"
+        exit 1
+    fi
+}
+
+
+# Upload a specific file and check it
+./sda-cli -config testing/s3cmd.conf upload data_file.c4gh
+check_uploaded_file "test/$user/data_file.c4gh" data_file.c4gh
+
+
+# Try to upload a file twice with the --force-overwrite flag
+output=$(./sda-cli -config testing/s3cmd.conf upload --force-overwrite data_file.c4gh)
+
+
+# Test recursive folder upload
+for k in data_file.c4gh data_file1.c4gh
+do
+    # Upload and check file
+    ./sda-cli -config testing/s3cmd.conf upload --force-overwrite "data_files_enc/$k"
+    check_uploaded_file "test/$user/$k" "$k"
+done
+
+
+
+# Upload a folder recursively and a single file
+./sda-cli -config testing/s3cmd.conf upload -r data_files_enc/dir1 data_files_enc/data_file3.c4gh
+
+# Check that files were uploaded with the local path prefix `data_files_enc` stripped from the target path
+for k in dir1/data_file.c4gh dir1/dir2/data_file.c4gh dir1/dir2/data_file2.c4gh data_file3.c4gh
+do
+    check_uploaded_file "test/$user/$k" "$k"
+done
+
+# Test upload to a different path
+
+# Upload a folder recursively and a single file in a specified upload folder
+uploadDir="testfolder"
+./sda-cli -config testing/s3cmd.conf upload -targetDir "$uploadDir" -r data_files_enc/dir1 data_files_enc/data_file3.c4gh
+
+# Do it again to test that we can pass -targetDir at the end
+./sda-cli -config testing/s3cmd.conf upload --force-overwrite -r data_files_enc/dir1 data_files_enc/data_file3.c4gh -targetDir "$uploadDir"
+
+# Check that files were uploaded with the local path prefix `data_files_enc` stripped from the
+# target path and into the specified upload folder
+for k in dir1/data_file.c4gh dir1/dir2/data_file.c4gh dir1/dir2/data_file2.c4gh data_file3.c4gh
+do
+    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
+done
+
+# Upload all contents of a folder recursively to a specified upload folder
+
+uploadDir="testfolder2"
+./sda-cli -config testing/s3cmd.conf upload -targetDir "$uploadDir" -r data_files_enc/dir1/.
+
+# Check that files were uploaded with the local path prefix `data_files_enc/dir1` stripped from the
+# target path and into the specified upload folder
+for k in data_file.c4gh dir2/data_file.c4gh dir2/data_file2.c4gh
+do
+    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
+done
+
+# Encrypt and upload
+
+mkdir data_files_unenc && mkdir data_files_unenc/dir1
+cp data_file data_files_unenc/. && cp data_file data_files_unenc/dir1/data_file1
+
+uploadDir="testEncryptUpload"
+./sda-cli -config testing/s3cmd.conf upload -encrypt-with-key sda_key.pub.pem -r data_files_unenc -targetDir "$uploadDir"
+
+check_encypted_file "data_files_unenc/data_file.c4gh" "data_files_unenc/dir1/data_file1.c4gh"
+
+for k in data_files_unenc/data_file.c4gh data_files_unenc/dir1/data_file1.c4gh
+do
+    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
+done
+
+if ! s3cmd -c testing/directS3 ls -r s3://test/"$user"/testEncryptUpload/data_files_unenc/ | grep -v -q 'c4gh'
+then
+    echo "No unencrypted files were uploaded during encrypt+upload"
+else
+    echo "Unencrypted files were uploaded during encrypt+upload"
+    exit 1
+fi
+
+echo "Integration tests for sda-cli upload finished successfully"
\ No newline at end of file
diff --git a/.github/integration/tests/30_list.sh b/.github/integration/tests/30_list.sh
new file mode 100755
index 00000000..544a45ed
--- /dev/null
+++ b/.github/integration/tests/30_list.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+if ./sda-cli -config testing/s3cmd.conf list | grep -q 'data_file.c4gh'
+then
+    echo "Listed file from s3 backend"
+else
+    echo "Failed to list file to s3 backend"
+    exit 1
+fi
+
+
+# Check listing files in a dataset
+output=$(./sda-cli -config testing/s3cmd-download.conf list -dataset https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080)
+expected="FileIDSizePathurn:neic:001-0011.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder/dummy_data.c4ghurn:neic:001-0021.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder2/dummy_data2.c4ghurn:neic:001-0031.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder2/random/dummy_data3.c4ghDatasetsize:3.1MB"
+if [[ "${output//[$' \t\n\r']/}" == "${expected//[$' \t\n\r']/}" ]];  then
+    echo "Successfully listed files in dataset"
+else
+    echo "Failed to list files in dataset"
+    exit 1
+fi
+
+# Check listing datasets
+output=$(./sda-cli -config testing/s3cmd-download.conf list --datasets -url http://localhost:8080)
+expected="https://doi.example/ty009.sfrrss/600.45asasga"
+if [[ $output == *"$expected"* ]]; then
+    echo "Successfully listed datasets"
+else
+    echo "Failed to list datasets"
+    exit 1
+fi
+
+echo "Integration tests for sda-cli list finished successfully"
\ No newline at end of file
diff --git a/.github/integration/tests/40_download.sh b/.github/integration/tests/40_download.sh
new file mode 100755
index 00000000..398c7410
--- /dev/null
+++ b/.github/integration/tests/40_download.sh
@@ -0,0 +1,128 @@
+#!/bin/bash
+set -e
+
+# Download file by using the sda-cli download command
+./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir test-download main/subfolder/dummy_data.c4gh
+
+# Check if file exists in the path
+if [ ! -f "test-download/main/subfolder/dummy_data" ]; then
+    echo "Downloaded file not found"
+    exit 1
+fi
+
+# Check the first line of that file
+first_line=$(head -n 1 test-download/main/subfolder/dummy_data)
+if [[ $first_line != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
+    echo "First line does not contain the expected string"
+    exit 1
+fi
+
+rm -r test-download
+
+# Download whole dataset by using the sda-cli download command
+./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-dataset --dataset
+
+filepaths="download-dataset/main/subfolder/dummy_data download-dataset/main/subfolder2/dummy_data2 download-dataset/main/subfolder2/random/dummy_data3"
+
+# Check if all the files of the dataset have been downloaded
+for filepath in $filepaths; do
+    if [ ! -f "$filepath" ]; then
+        echo "File $filepath does not exist"
+        exit 1
+    fi
+done
+
+rm -r download-dataset
+
+# Download encrypted file by using the sda-cli download comand
+# Create a user key pair
+if ( yes "" | ./sda-cli createKey user_key ) ; then
+    echo "Created a user key pair for downloading encrypted files"
+else
+    echo "Failed to create a user key pair for downloading encrypted files"
+    exit 1
+fi
+./sda-cli -config testing/s3cmd-download.conf download -pubkey user_key.pub.pem -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir test-download main/subfolder/dummy_data.c4gh
+
+# check if file exists in the path
+if [ ! -f "test-download/main/subfolder/dummy_data.c4gh" ]; then
+    echo "Downloaded file not found"
+    exit 1
+fi
+
+# decrypt the downloaded file
+C4GH_PASSWORD="" ./sda-cli decrypt -key user_key.sec.pem test-download/main/subfolder/dummy_data.c4gh
+
+if [ -f test-download/main/subfolder/dummy_data  ]; then
+    echo "Decrypting downloaded file succeeded"
+else
+    echo "Failed to decrypt downloaded file"
+    exit 1
+fi
+
+# check the first line of that file
+first_line=$(head -n 1 test-download/main/subfolder/dummy_data)
+if [[ $first_line != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
+    echo "First line does not contain the expected string"
+    exit 1
+fi
+
+# Download recursively a folder
+echo "Downloading content of folder"
+./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-folder --recursive main/subfolder2
+
+folderpaths="download-folder/main/subfolder2/dummy_data2 download-folder/main/subfolder2/random/dummy_data3"
+
+# Check if the content of the folder has been downloaded
+for folderpath in $folderpaths; do
+    if [ ! -f "$folderpath" ]; then
+        echo "Content of folder $folderpath is missing"
+        exit 1
+    fi
+done
+
+rm -r download-folder
+
+# Download file by providing the file id
+./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-fileid urn:neic:001-001
+
+# Check if file exists in the path
+if [ ! -f "download-fileid/main/subfolder/dummy_data" ]; then
+    echo "Downloaded file by using the file id not found"
+    exit 1
+fi
+
+# Check the first line of the file
+first_line_id=$(head -n 1 download-fileid/main/subfolder/dummy_data)
+if [[ $first_line_id != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
+    echo "This is not the file with the given file id"
+    exit 1
+fi
+
+rm -r download-fileid
+
+# Download the file paths content of a text file
+echo "Downloading content of a text file"
+./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-from-file --from-file testing/file-list.txt
+
+# Check if the content of the text file has been downloaded
+content_paths="download-from-file/main/subfolder/dummy_data download-from-file/main/subfolder2/dummy_data2"
+
+for content_path in $content_paths; do
+    if [ ! -f "$content_path" ]; then
+        echo "Content of the text file $content_path is missing"
+        exit 1
+    fi
+done
+
+# Check the first line of the file
+first_line_file=$(head -n 1 download-from-file/main/subfolder/dummy_data)
+if [[ $first_line_file != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
+    echo "First line does not contain the expected string"
+    exit 1
+fi
+
+rm -r download-from-file
+rm -r test-download
+
+echo "Integration tests for sda-cli download finished successfully"
\ No newline at end of file
diff --git a/.github/integration/tests/tests.sh b/.github/integration/tests/tests.sh
deleted file mode 100755
index 618116fa..00000000
--- a/.github/integration/tests/tests.sh
+++ /dev/null
@@ -1,373 +0,0 @@
-#!/bin/bash
-set -e
-
-# Function checking that a file was encrypted
-function check_encypted_file() {
-
-    for k in $1
-    do
-        output=$(head -c 8 "$k")
-
-        if [[ "$output" = "crypt4gh"  ]]; then
-            echo "Encrypted data file"
-        else
-            echo "Failed to encrypt file"
-            exit 1
-        fi
-    done
-}
-
-# Function checking that a file was uploaded to the S3 backend
-function check_uploaded_file() {
-    output=$(s3cmd -c testing/directS3 ls s3://"$1" | grep -q "$2")
-    if $output ; then
-        echo "Uploaded encrypted file to s3 backend"
-    else
-        echo "Failed to upload file to s3 backend"
-        exit 1
-    fi
-}
-
-# inferred from access_key in testing/s3cmd.conf
-user=test_dummy.org
-
-# Create random file
-dd if=/dev/urandom of=data_file count=1 bs=1M
-
-# Create key pair
-if ( yes "" | ./sda-cli createKey sda_key ) ; then
-    echo "Created key pair for encryption"
-else
-    echo "Failed to create key pair for encryption"
-    exit 1
-fi
-
-# Encrypt a file
-./sda-cli encrypt -key sda_key.pub.pem data_file
-
-files="data_file.c4gh"
-check_encypted_file $files
-
-# Upload a specific file and check it
-./sda-cli -config testing/s3cmd.conf upload data_file.c4gh
-check_uploaded_file "test/$user/data_file.c4gh" data_file.c4gh
-
-
-if ./sda-cli -config testing/s3cmd.conf list | grep -q 'data_file.c4gh'
-then
-    echo "Listed file from s3 backend"
-else
-    echo "Failed to list file to s3 backend"
-    exit 1
-fi
-
-# Try to upload a file twice with the --force-overwrite flag
-output=$(./sda-cli -config testing/s3cmd.conf upload --force-overwrite data_file.c4gh)
-
-# Create and encrypt multiple files in a folder
-
-# Create folder and encrypt files in it
-cp data_file data_file1
-mkdir data_files_enc
-./sda-cli encrypt -key sda_key.pub.pem -outdir data_files_enc data_file data_file1
-
-check_encypted_file "data_files_enc/data_file.c4gh data_files_enc/data_file1.c4gh"
-
-for k in data_file.c4gh data_file1.c4gh
-do
-    # Upload and check file
-    ./sda-cli -config testing/s3cmd.conf upload --force-overwrite "data_files_enc/$k"
-    check_uploaded_file "test/$user/$k" "$k"
-done
-
-# Test recursive folder upload
-
-# Create folder with subfolder structure and add some encrypted files
-mkdir data_files_enc/dir1 data_files_enc/dir1/dir2
-cp data_files_enc/data_file.c4gh data_files_enc/data_file3.c4gh
-cp data_files_enc/data_file.c4gh data_files_enc/dir1/data_file.c4gh
-cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file.c4gh
-cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file2.c4gh
-
-# Upload a folder recursively and a single file
-./sda-cli -config testing/s3cmd.conf upload -r data_files_enc/dir1 data_files_enc/data_file3.c4gh
-
-# Check that files were uploaded with the local path prefix `data_files_enc` stripped from the target path
-for k in dir1/data_file.c4gh dir1/dir2/data_file.c4gh dir1/dir2/data_file2.c4gh data_file3.c4gh
-do
-    check_uploaded_file "test/$user/$k" "$k"
-done
-
-# Test upload to a different path
-
-# Upload a folder recursively and a single file in a specified upload folder
-uploadDir="testfolder"
-./sda-cli -config testing/s3cmd.conf upload -targetDir "$uploadDir" -r data_files_enc/dir1 data_files_enc/data_file3.c4gh
-
-# Do it again to test that we can pass -targetDir at the end
-./sda-cli -config testing/s3cmd.conf upload --force-overwrite -r data_files_enc/dir1 data_files_enc/data_file3.c4gh -targetDir "$uploadDir"
-
-# Check that files were uploaded with the local path prefix `data_files_enc` stripped from the
-# target path and into the specified upload folder
-for k in dir1/data_file.c4gh dir1/dir2/data_file.c4gh dir1/dir2/data_file2.c4gh data_file3.c4gh
-do
-    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
-done
-
-# Upload all contents of a folder recursively to a specified upload folder
-
-uploadDir="testfolder2"
-./sda-cli -config testing/s3cmd.conf upload -targetDir "$uploadDir" -r data_files_enc/dir1/.
-
-# Check that files were uploaded with the local path prefix `data_files_enc/dir1` stripped from the
-# target path and into the specified upload folder
-for k in data_file.c4gh dir2/data_file.c4gh dir2/data_file2.c4gh
-do
-    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
-done
-
-# Encrypt and upload
-
-mkdir data_files_unenc && mkdir data_files_unenc/dir1
-cp data_file data_files_unenc/. && cp data_file data_files_unenc/dir1/data_file1
-
-uploadDir="testEncryptUpload"
-./sda-cli -config testing/s3cmd.conf upload -encrypt-with-key sda_key.pub.pem -r data_files_unenc -targetDir "$uploadDir"
-
-check_encypted_file "data_files_unenc/data_file.c4gh" "data_files_unenc/dir1/data_file1.c4gh"
-
-for k in data_files_unenc/data_file.c4gh data_files_unenc/dir1/data_file1.c4gh
-do
-    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
-done
-
-if ! s3cmd -c testing/directS3 ls -r s3://test/"$user"/testEncryptUpload/data_files_unenc/ | grep -v -q 'c4gh'
-then
-    echo "No unencrypted files were uploaded during encrypt+upload"
-else
-    echo "Unencrypted files were uploaded during encrypt+upload"
-    exit 1
-fi
-
-# Download file by using the sda download service
-./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir test-download main/subfolder/dummy_data.c4gh
-
-# Check if file exists in the path
-if [ ! -f "test-download/main/subfolder/dummy_data" ]; then
-    echo "Downloaded file not found"
-    exit 1
-fi
-
-# Check the first line of that file
-first_line=$(head -n 1 test-download/main/subfolder/dummy_data)
-if [[ $first_line != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
-    echo "First line does not contain the expected string"
-    exit 1
-fi
-
-rm -r test-download
-
-# Check listing files in a dataset
-output=$(./sda-cli -config testing/s3cmd-download.conf list -dataset https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080)
-expected="FileIDSizePathurn:neic:001-0011.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder/dummy_data.c4ghurn:neic:001-0021.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder2/dummy_data2.c4ghurn:neic:001-0031.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder2/random/dummy_data3.c4ghDatasetsize:3.1MB"
-if [[ "${output//[$' \t\n\r']/}" == "${expected//[$' \t\n\r']/}" ]];  then
-    echo "Successfully listed files in dataset"
-else
-    echo "Failed to list files in dataset"
-    exit 1
-fi
-
-# Check listing datasets
-output=$(./sda-cli -config testing/s3cmd-download.conf list --datasets -url http://localhost:8080)
-expected="https://doi.example/ty009.sfrrss/600.45asasga"
-if [[ $output == *"$expected"* ]]; then
-    echo "Successfully listed datasets"
-else
-    echo "Failed to list datasets"
-    exit 1
-fi
-
-# Download whole dataset by using the sda-download feature
-./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-dataset --dataset
-
-filepaths="download-dataset/main/subfolder/dummy_data download-dataset/main/subfolder2/dummy_data2 download-dataset/main/subfolder2/random/dummy_data3"
-
-# Check if all the files of the dataset have been downloaded
-for filepath in $filepaths; do
-    if [ ! -f "$filepath" ]; then
-        echo "File $filepath does not exist"
-        exit 1
-    fi
-done
-
-rm -r download-dataset
-
-# Download encrypted file by using the sda download service
-# Create a user key pair
-if ( yes "" | ./sda-cli createKey user_key ) ; then
-    echo "Created a user key pair for downloading encrypted files"
-else
-    echo "Failed to create a user key pair for downloading encrypted files"
-    exit 1
-fi
-./sda-cli -config testing/s3cmd-download.conf download -pubkey user_key.pub.pem -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir test-download main/subfolder/dummy_data.c4gh
-
-# check if file exists in the path
-if [ ! -f "test-download/main/subfolder/dummy_data.c4gh" ]; then
-    echo "Downloaded file not found"
-    exit 1
-fi
-
-# decrypt the downloaded file
-C4GH_PASSWORD="" ./sda-cli decrypt -key user_key.sec.pem test-download/main/subfolder/dummy_data.c4gh
-
-if [ -f test-download/main/subfolder/dummy_data  ]; then
-    echo "Decrypting downloaded file succeeded"
-else
-    echo "Failed to decrypt downloaded file"
-    exit 1
-fi
-
-# check the first line of that file
-first_line=$(head -n 1 test-download/main/subfolder/dummy_data)
-if [[ $first_line != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
-    echo "First line does not contain the expected string"
-    exit 1
-fi
-
-# Test multiple pub key encryption
-
-# Create another couple of key-pairs
-for c in 1 2
-do
-    if ( yes "" | ./sda-cli createKey sda_key$c ) ; then
-        echo "Created key pair for encryption"
-    else
-        echo "Failed to create key pair for encryption"
-        exit 1
-fi
-done
-
-# Create file with concatenated pub keys
-cat sda_key1.pub.pem sda_key2.pub.pem > sda_keys
-
-# Create test files
-cp test-download/main/subfolder/dummy_data data_file_keys
-
-# Encrypt with multiple key flag calls
-./sda-cli encrypt -key sda_key.pub.pem -key sda_key2.pub.pem data_file_keys
-check_encypted_file "data_file_keys.c4gh"
-
-# Decrypt file with both keys
-for key in sda_key sda_key2
-do
-    rm data_file_keys
-    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
-    if [ -f data_file_keys ]; then
-        echo "Decrypted data file"
-    else
-        echo "Failed to decrypt data file with $key"
-        exit 1
-    fi
-done
-rm data_file_keys.c4gh
-
-# Encrypt with concatenated key file
-./sda-cli encrypt -key sda_keys data_file_keys
-check_encypted_file "data_file_keys.c4gh"
-
-# Decrypt file with both keys
-for key in sda_key1 sda_key2
-do
-    rm data_file_keys
-    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
-    if [ -f data_file_keys ]; then
-        echo "Decrypted data file"
-    else
-        echo "Failed to decrypt data file with $key"
-        exit 1
-    fi
-done
-rm data_file_keys.c4gh
-
-# Encrypt with concatenated key file and a key flag call
-./sda-cli encrypt -key sda_key.pub.pem -key sda_keys data_file_keys
-check_encypted_file "data_file_keys.c4gh"
-
-# Decrypt file with all keys
-for key in sda_key sda_key1 sda_key2
-do
-    rm data_file_keys
-    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
-    if [ -f data_file_keys ]; then
-        echo "Decrypted data file"
-    else
-        echo "Failed to decrypt data file with $key"
-        exit 1
-    fi
-done
-
-# Remove files used for encrypt and upload
-rm -r data_files_enc
-rm -r data_files_unenc
-rm sda_key* data_file*
-rm -r test-download
-
-# Download recursively a folder
-echo "Downloading content of folder"
-./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-folder --recursive main/subfolder2
-
-folderpaths="download-folder/main/subfolder2/dummy_data2 download-folder/main/subfolder2/random/dummy_data3"
-
-# Check if the content of the folder has been downloaded
-for folderpath in $folderpaths; do
-    if [ ! -f "$folderpath" ]; then
-        echo "Content of folder $folderpath is missing"
-        exit 1
-    fi
-done
-
-rm -r download-folder
-
-# Download file by providing the file id
-./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-fileid urn:neic:001-001
-
-# Check if file exists in the path
-if [ ! -f "download-fileid/main/subfolder/dummy_data" ]; then
-    echo "Downloaded file by using the file id not found"
-    exit 1
-fi
-
-# Check the first line of the file
-first_line_id=$(head -n 1 download-fileid/main/subfolder/dummy_data)
-if [[ $first_line_id != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
-    echo "This is not the file with the given file id"
-    exit 1
-fi
-
-rm -r download-fileid
-
-# Download the file paths content of a text file
-echo "Downloading content of a text file"
-./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-from-file --from-file testing/file-list.txt
-
-# Check if the content of the text file has been downloaded
-content_paths="download-from-file/main/subfolder/dummy_data download-from-file/main/subfolder2/dummy_data2"
-
-for content_path in $content_paths; do
-    if [ ! -f "$content_path" ]; then
-        echo "Content of the text file $content_path is missing"
-        exit 1
-    fi
-done
-
-# Check the first line of the file
-first_line_file=$(head -n 1 download-from-file/main/subfolder/dummy_data)
-if [[ $first_line_file != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
-    echo "First line does not contain the expected string"
-    exit 1
-fi
-
-rm -r download-from-file
-
-echo "Integration test finished successfully"

From 7f70b3a207d01c6e64a2ecfe7474df7f0886d36f Mon Sep 17 00:00:00 2001
From: MalinAhlberg <ahlberg.malin@gmail.com>
Date: Tue, 17 Dec 2024 14:33:39 +0100
Subject: [PATCH 2/5] [tests] run all test files with github actions

---
 .github/workflows/integration.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index c49d993a..0868eb30 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -25,4 +25,8 @@ jobs:
         run: bash -x .github/integration/setup/setup.sh
 
       - name: Run tests
-        run: bash -x .github/integration/tests/tests.sh
+        run: |
+            ls -1 .github/integration/tests/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do
+              echo "Executing setup script $runscript";
+              bash -x "$runscript";
+            done
\ No newline at end of file

From 37279469e4864bfe3a23cd94ab39f7ccfdbdf2b6 Mon Sep 17 00:00:00 2001
From: MalinAhlberg <ahlberg.malin@gmail.com>
Date: Wed, 18 Dec 2024 09:05:21 +0100
Subject: [PATCH 3/5] [tests] update comments

---
 .github/integration/tests/10_encrypt_decrypt.sh |  8 +++-----
 .github/integration/tests/20_upload.sh          |  2 +-
 .github/integration/tests/40_download.sh        | 10 +++++-----
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/.github/integration/tests/10_encrypt_decrypt.sh b/.github/integration/tests/10_encrypt_decrypt.sh
index 49af6d87..ba4a8761 100755
--- a/.github/integration/tests/10_encrypt_decrypt.sh
+++ b/.github/integration/tests/10_encrypt_decrypt.sh
@@ -38,8 +38,6 @@ files="data_file.c4gh"
 check_encypted_file $files
 
 
-# Create and encrypt multiple files in a folder
-
 # Create folder and encrypt files in it
 cp data_file data_file1
 mkdir data_files_enc
@@ -78,7 +76,7 @@ cp data_file data_file_keys
 # Encrypt with multiple key flag calls
 ./sda-cli encrypt -key sda_key.pub.pem -key sda_key2.pub.pem data_file_keys
 check_encypted_file "data_file_keys.c4gh"
-# Decrypt file with both keys
+# Decrypt file with both keys, one at the time
 for key in sda_key sda_key2
 do
     rm data_file_keys
@@ -94,11 +92,11 @@ echo "Could decrypt with both keys from multiple key flag"
 rm data_file_keys.c4gh
 
 
-# Encrypt with concatenated key file and a key flag call
+# Encrypt with a single key and with a concatenated key file
 ./sda-cli encrypt -key sda_key.pub.pem -key sda_keys data_file_keys
 check_encypted_file "data_file_keys.c4gh"
 
-# Decrypt file with both keys
+# Decrypt file with all three keys
 for key in sda_key sda_key1 sda_key2
 do
     rm data_file_keys
diff --git a/.github/integration/tests/20_upload.sh b/.github/integration/tests/20_upload.sh
index dd6f0994..f2f4a5f2 100755
--- a/.github/integration/tests/20_upload.sh
+++ b/.github/integration/tests/20_upload.sh
@@ -42,7 +42,7 @@ check_uploaded_file "test/$user/data_file.c4gh" data_file.c4gh
 output=$(./sda-cli -config testing/s3cmd.conf upload --force-overwrite data_file.c4gh)
 
 
-# Test recursive folder upload
+# Test upload all files from a folder, one by one
 for k in data_file.c4gh data_file1.c4gh
 do
     # Upload and check file
diff --git a/.github/integration/tests/40_download.sh b/.github/integration/tests/40_download.sh
index 398c7410..1176c036 100755
--- a/.github/integration/tests/40_download.sh
+++ b/.github/integration/tests/40_download.sh
@@ -34,7 +34,6 @@ done
 
 rm -r download-dataset
 
-# Download encrypted file by using the sda-cli download comand
 # Create a user key pair
 if ( yes "" | ./sda-cli createKey user_key ) ; then
     echo "Created a user key pair for downloading encrypted files"
@@ -42,15 +41,16 @@ else
     echo "Failed to create a user key pair for downloading encrypted files"
     exit 1
 fi
+# Download encrypted file by using the sda-cli download command
 ./sda-cli -config testing/s3cmd-download.conf download -pubkey user_key.pub.pem -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir test-download main/subfolder/dummy_data.c4gh
 
-# check if file exists in the path
+# Check if file exists in the path
 if [ ! -f "test-download/main/subfolder/dummy_data.c4gh" ]; then
     echo "Downloaded file not found"
     exit 1
 fi
 
-# decrypt the downloaded file
+# Decrypt the downloaded file
 C4GH_PASSWORD="" ./sda-cli decrypt -key user_key.sec.pem test-download/main/subfolder/dummy_data.c4gh
 
 if [ -f test-download/main/subfolder/dummy_data  ]; then
@@ -60,7 +60,7 @@ else
     exit 1
 fi
 
-# check the first line of that file
+# Check the first line of that file
 first_line=$(head -n 1 test-download/main/subfolder/dummy_data)
 if [[ $first_line != *"THIS FILE IS JUST DUMMY DATA"* ]]; then
     echo "First line does not contain the expected string"
@@ -83,7 +83,7 @@ done
 
 rm -r download-folder
 
-# Download file by providing the file id
+# Download dataset by providing the dataset id
 ./sda-cli -config testing/s3cmd-download.conf download -dataset-id https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080 -outdir download-fileid urn:neic:001-001
 
 # Check if file exists in the path

From 0d9f16c2e385d1d6c2a161cf5e12dc6d10d53bec Mon Sep 17 00:00:00 2001
From: MalinAhlberg <ahlberg.malin@gmail.com>
Date: Wed, 18 Dec 2024 09:05:35 +0100
Subject: [PATCH 4/5] [tests] move set up to correct file

---
 .github/integration/tests/10_encrypt_decrypt.sh | 8 --------
 .github/integration/tests/20_upload.sh          | 8 ++++++++
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/integration/tests/10_encrypt_decrypt.sh b/.github/integration/tests/10_encrypt_decrypt.sh
index ba4a8761..a2fae75f 100755
--- a/.github/integration/tests/10_encrypt_decrypt.sh
+++ b/.github/integration/tests/10_encrypt_decrypt.sh
@@ -45,14 +45,6 @@ mkdir data_files_enc
 
 check_encypted_file "data_files_enc/data_file.c4gh data_files_enc/data_file1.c4gh"
 
-
-# Create folder with subfolder structure and add some encrypted files
-mkdir data_files_enc/dir1 data_files_enc/dir1/dir2
-cp data_files_enc/data_file.c4gh data_files_enc/data_file3.c4gh
-cp data_files_enc/data_file.c4gh data_files_enc/dir1/data_file.c4gh
-cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file.c4gh
-cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file2.c4gh
-
 # Test multiple pub key encryption
 
 # Create another couple of key-pairs
diff --git a/.github/integration/tests/20_upload.sh b/.github/integration/tests/20_upload.sh
index f2f4a5f2..ec929b9e 100755
--- a/.github/integration/tests/20_upload.sh
+++ b/.github/integration/tests/20_upload.sh
@@ -33,6 +33,14 @@ function check_uploaded_file() {
 }
 
 
+# Create folder with subfolder structure and add some encrypted files
+mkdir data_files_enc/dir1 data_files_enc/dir1/dir2
+cp data_files_enc/data_file.c4gh data_files_enc/data_file3.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/data_file.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file2.c4gh
+
+
 # Upload a specific file and check it
 ./sda-cli -config testing/s3cmd.conf upload data_file.c4gh
 check_uploaded_file "test/$user/data_file.c4gh" data_file.c4gh

From 28fb8272edb373bb3d8b3d4c93cad1dd09ad5280 Mon Sep 17 00:00:00 2001
From: MalinAhlberg <ahlberg.malin@gmail.com>
Date: Fri, 20 Dec 2024 09:38:47 +0100
Subject: [PATCH 5/5] [tests] move checker functions to separate file

also clean up minor typos etc
---
 .github/integration/scripts/checkers.sh       | 28 +++++++++++++++
 .../integration/tests/10_encrypt_decrypt.sh   | 34 +++++-------------
 .github/integration/tests/20_upload.sh        | 36 +++----------------
 3 files changed, 41 insertions(+), 57 deletions(-)
 create mode 100755 .github/integration/scripts/checkers.sh

diff --git a/.github/integration/scripts/checkers.sh b/.github/integration/scripts/checkers.sh
new file mode 100755
index 00000000..9a36e25f
--- /dev/null
+++ b/.github/integration/scripts/checkers.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+# Function checking that file(s) were encrypted
+function check_encrypted_file {
+    for k do
+        echo "working with $k"
+        output=$(head -c 8 "$k")
+
+        if [[ "$output" == "crypt4gh" ]]; then
+            echo "Encrypted data file: $k"
+        else
+            echo "Failed to encrypt file: $k"
+            exit 1
+        fi
+    done
+}
+
+
+# Function checking that a file was uploaded to the S3 backend
+function check_uploaded_file {
+    if s3cmd -c testing/directS3 ls s3://"$1" | grep -q "$2"; then
+        echo "Uploaded encrypted file to s3 backend"
+    else
+        echo "Failed to upload file to s3 backend"
+        exit 1
+    fi
+}
diff --git a/.github/integration/tests/10_encrypt_decrypt.sh b/.github/integration/tests/10_encrypt_decrypt.sh
index a2fae75f..4607ce1b 100755
--- a/.github/integration/tests/10_encrypt_decrypt.sh
+++ b/.github/integration/tests/10_encrypt_decrypt.sh
@@ -1,24 +1,7 @@
 #!/bin/bash
 set -e
-
-# Function checking that a file was encrypted
-function check_encypted_file() {
-
-    for k in $1
-    do
-        output=$(head -c 8 "$k")
-
-        if [[ "$output" = "crypt4gh"  ]]; then
-            echo "Encrypted data file"
-        else
-            echo "Failed to encrypt file"
-            exit 1
-        fi
-    done
-}
-
-# inferred from access_key in testing/s3cmd.conf
-user=test_dummy.org
+test_dir=$(dirname "$0")
+source "$test_dir/../scripts/checkers.sh"
 
 # Create random file
 dd if=/dev/urandom of=data_file count=1 bs=1M
@@ -34,8 +17,7 @@ fi
 # Encrypt a file
 ./sda-cli encrypt -key sda_key.pub.pem data_file
 
-files="data_file.c4gh"
-check_encypted_file $files
+check_encrypted_file data_file.c4gh
 
 
 # Create folder and encrypt files in it
@@ -43,7 +25,7 @@ cp data_file data_file1
 mkdir data_files_enc
 ./sda-cli encrypt -key sda_key.pub.pem -outdir data_files_enc data_file data_file1
 
-check_encypted_file "data_files_enc/data_file.c4gh data_files_enc/data_file1.c4gh"
+check_encrypted_file data_files_enc/data_file.c4gh data_files_enc/data_file1.c4gh
 
 # Test multiple pub key encryption
 
@@ -55,7 +37,7 @@ do
     else
         echo "Failed to create key pair for encryption"
         exit 1
-fi
+    fi
 done
 
 # Create file with concatenated pub keys
@@ -67,7 +49,7 @@ cp data_file data_file_keys
 
 # Encrypt with multiple key flag calls
 ./sda-cli encrypt -key sda_key.pub.pem -key sda_key2.pub.pem data_file_keys
-check_encypted_file "data_file_keys.c4gh"
+check_encrypted_file data_file_keys.c4gh
 # Decrypt file with both keys, one at the time
 for key in sda_key sda_key2
 do
@@ -86,7 +68,7 @@ rm data_file_keys.c4gh
 
 # Encrypt with a single key and with a concatenated key file
 ./sda-cli encrypt -key sda_key.pub.pem -key sda_keys data_file_keys
-check_encypted_file "data_file_keys.c4gh"
+check_encrypted_file data_file_keys.c4gh
 
 # Decrypt file with all three keys
 for key in sda_key sda_key1 sda_key2
@@ -105,7 +87,7 @@ rm data_file_keys.c4gh
 
 # Encrypt with concatenated key file
 ./sda-cli encrypt -key sda_keys data_file_keys
-check_encypted_file "data_file_keys.c4gh"
+check_encrypted_file data_file_keys.c4gh
 
 # Decrypt file with all keys
 for key in sda_key1 sda_key2
diff --git a/.github/integration/tests/20_upload.sh b/.github/integration/tests/20_upload.sh
index ec929b9e..2a16f5a0 100755
--- a/.github/integration/tests/20_upload.sh
+++ b/.github/integration/tests/20_upload.sh
@@ -1,38 +1,12 @@
 #!/bin/bash
 set -e
+test_dir=$(dirname "$0")
+source "$test_dir/../scripts/checkers.sh"
+
 # inferred from access_key in testing/s3cmd.conf
 user=test_dummy.org
 
 
-# Function checking that a file was encrypted
-function check_encypted_file() {
-
-    for k in $1
-    do
-        output=$(head -c 8 "$k")
-
-        if [[ "$output" = "crypt4gh"  ]]; then
-            echo "Encrypted data file"
-        else
-            echo "Failed to encrypt file"
-            exit 1
-        fi
-    done
-}
-
-# Function checking that a file was uploaded to the S3 backend
-function check_uploaded_file() {
-    # TODO if emtpy, this will fail silently
-    output=$(s3cmd -c testing/directS3 ls s3://"$1" | grep -q "$2")
-    if $output ; then
-        echo "Uploaded encrypted file to s3 backend"
-    else
-        echo "Failed to upload file to s3 backend"
-        exit 1
-    fi
-}
-
-
 # Create folder with subfolder structure and add some encrypted files
 mkdir data_files_enc/dir1 data_files_enc/dir1/dir2
 cp data_files_enc/data_file.c4gh data_files_enc/data_file3.c4gh
@@ -47,7 +21,7 @@ check_uploaded_file "test/$user/data_file.c4gh" data_file.c4gh
 
 
 # Try to upload a file twice with the --force-overwrite flag
-output=$(./sda-cli -config testing/s3cmd.conf upload --force-overwrite data_file.c4gh)
+./sda-cli -config testing/s3cmd.conf upload --force-overwrite data_file.c4gh
 
 
 # Test upload all files from a folder, one by one
@@ -105,7 +79,7 @@ cp data_file data_files_unenc/. && cp data_file data_files_unenc/dir1/data_file1
 uploadDir="testEncryptUpload"
 ./sda-cli -config testing/s3cmd.conf upload -encrypt-with-key sda_key.pub.pem -r data_files_unenc -targetDir "$uploadDir"
 
-check_encypted_file "data_files_unenc/data_file.c4gh" "data_files_unenc/dir1/data_file1.c4gh"
+check_encrypted_file data_files_unenc/data_file.c4gh data_files_unenc/dir1/data_file1.c4gh
 
 for k in data_files_unenc/data_file.c4gh data_files_unenc/dir1/data_file1.c4gh
 do