split integration tests (#484)

.github/integration/scripts/checkers.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+# Function checking that file(s) were encrypted
+function check_encrypted_file {
+    for k do
+        echo "working with $k"
+        output=$(head -c 8 "$k")
+
+        if [[ "$output" == "crypt4gh" ]]; then
+            echo "Encrypted data file: $k"
+        else
+            echo "Failed to encrypt file: $k"
+            exit 1
+        fi
+    done
+}
+
+
+# Function checking that a file was uploaded to the S3 backend
+function check_uploaded_file {
+    if s3cmd -c testing/directS3 ls s3://"$1" | grep -q "$2"; then
+        echo "Uploaded encrypted file to s3 backend"
+    else
+        echo "Failed to upload file to s3 backend"
+        exit 1
+    fi
+}

.github/integration/tests/10_encrypt_decrypt.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+set -e
+test_dir=$(dirname "$0")
+source "$test_dir/../scripts/checkers.sh"
+
+# Create random file
+dd if=/dev/urandom of=data_file count=1 bs=1M
+
+# Create key pair
+if ( yes "" | ./sda-cli createKey sda_key ) ; then
+    echo "Created key pair for encryption"
+else
+    echo "Failed to create key pair for encryption"
+    exit 1
+fi
+
+# Encrypt a file
+./sda-cli encrypt -key sda_key.pub.pem data_file
+
+check_encrypted_file data_file.c4gh
+
+
+# Create folder and encrypt files in it
+cp data_file data_file1
+mkdir data_files_enc
+./sda-cli encrypt -key sda_key.pub.pem -outdir data_files_enc data_file data_file1
+
+check_encrypted_file data_files_enc/data_file.c4gh data_files_enc/data_file1.c4gh
+
+# Test multiple pub key encryption
+
+# Create another couple of key-pairs
+for c in 1 2
+do
+    if ( yes "" | ./sda-cli createKey sda_key$c ) ; then
+        echo "Created key pair for encryption"
+    else
+        echo "Failed to create key pair for encryption"
+        exit 1
+    fi
+done
+
+# Create file with concatenated pub keys
+cat sda_key1.pub.pem sda_key2.pub.pem > sda_keys
+
+
+# Create test files
+cp data_file data_file_keys
+
+# Encrypt with multiple key flag calls
+./sda-cli encrypt -key sda_key.pub.pem -key sda_key2.pub.pem data_file_keys
+check_encrypted_file data_file_keys.c4gh
+# Decrypt file with both keys, one at the time
+for key in sda_key sda_key2
+do
+    rm data_file_keys
+    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
+    if [ -f data_file_keys ]; then
+        echo "Decrypted data file"
+    else
+        echo "Failed to decrypt data file with $key"
+        exit 1
+    fi
+done
+echo "Could decrypt with both keys from multiple key flag"
+rm data_file_keys.c4gh
+
+
+# Encrypt with a single key and with a concatenated key file
+./sda-cli encrypt -key sda_key.pub.pem -key sda_keys data_file_keys
+check_encrypted_file data_file_keys.c4gh
+
+# Decrypt file with all three keys
+for key in sda_key sda_key1 sda_key2
+do
+    rm data_file_keys
+    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
+    if [ -f data_file_keys ]; then
+        echo "Decrypted data file"
+    else
+        echo "Failed to decrypt data file with $key"
+        exit 1
+    fi
+done
+echo "Could decrypt with both keys from concatenated key"
+rm data_file_keys.c4gh
+
+# Encrypt with concatenated key file
+./sda-cli encrypt -key sda_keys data_file_keys
+check_encrypted_file data_file_keys.c4gh
+
+# Decrypt file with all keys
+for key in sda_key1 sda_key2
+do
+    rm data_file_keys
+    C4GH_PASSWORD="" ./sda-cli decrypt -key $key.sec.pem data_file_keys.c4gh
+    if [ -f data_file_keys ]; then
+        echo "Decrypted data file"
+    else
+        echo "Failed to decrypt data file with $key"
+        exit 1
+    fi
+done
+
+echo "Could decrypt with all keys from concatenated key"
+
+echo "Integration tests for sda-cli encrypt finished successfully"

.github/integration/tests/20_upload.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+set -e
+test_dir=$(dirname "$0")
+source "$test_dir/../scripts/checkers.sh"
+
+# inferred from access_key in testing/s3cmd.conf
+user=test_dummy.org
+
+
+# Create folder with subfolder structure and add some encrypted files
+mkdir data_files_enc/dir1 data_files_enc/dir1/dir2
+cp data_files_enc/data_file.c4gh data_files_enc/data_file3.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/data_file.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file.c4gh
+cp data_files_enc/data_file.c4gh data_files_enc/dir1/dir2/data_file2.c4gh
+
+
+# Upload a specific file and check it
+./sda-cli -config testing/s3cmd.conf upload data_file.c4gh
+check_uploaded_file "test/$user/data_file.c4gh" data_file.c4gh
+
+
+# Try to upload a file twice with the --force-overwrite flag
+./sda-cli -config testing/s3cmd.conf upload --force-overwrite data_file.c4gh
+
+
+# Test upload all files from a folder, one by one
+for k in data_file.c4gh data_file1.c4gh
+do
+    # Upload and check file
+    ./sda-cli -config testing/s3cmd.conf upload --force-overwrite "data_files_enc/$k"
+    check_uploaded_file "test/$user/$k" "$k"
+done
+
+
+
+# Upload a folder recursively and a single file
+./sda-cli -config testing/s3cmd.conf upload -r data_files_enc/dir1 data_files_enc/data_file3.c4gh
+
+# Check that files were uploaded with the local path prefix `data_files_enc` stripped from the target path
+for k in dir1/data_file.c4gh dir1/dir2/data_file.c4gh dir1/dir2/data_file2.c4gh data_file3.c4gh
+do
+    check_uploaded_file "test/$user/$k" "$k"
+done
+
+# Test upload to a different path
+
+# Upload a folder recursively and a single file in a specified upload folder
+uploadDir="testfolder"
+./sda-cli -config testing/s3cmd.conf upload -targetDir "$uploadDir" -r data_files_enc/dir1 data_files_enc/data_file3.c4gh
+
+# Do it again to test that we can pass -targetDir at the end
+./sda-cli -config testing/s3cmd.conf upload --force-overwrite -r data_files_enc/dir1 data_files_enc/data_file3.c4gh -targetDir "$uploadDir"
+
+# Check that files were uploaded with the local path prefix `data_files_enc` stripped from the
+# target path and into the specified upload folder
+for k in dir1/data_file.c4gh dir1/dir2/data_file.c4gh dir1/dir2/data_file2.c4gh data_file3.c4gh
+do
+    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
+done
+
+# Upload all contents of a folder recursively to a specified upload folder
+
+uploadDir="testfolder2"
+./sda-cli -config testing/s3cmd.conf upload -targetDir "$uploadDir" -r data_files_enc/dir1/.
+
+# Check that files were uploaded with the local path prefix `data_files_enc/dir1` stripped from the
+# target path and into the specified upload folder
+for k in data_file.c4gh dir2/data_file.c4gh dir2/data_file2.c4gh
+do
+    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
+done
+
+# Encrypt and upload
+
+mkdir data_files_unenc && mkdir data_files_unenc/dir1
+cp data_file data_files_unenc/. && cp data_file data_files_unenc/dir1/data_file1
+
+uploadDir="testEncryptUpload"
+./sda-cli -config testing/s3cmd.conf upload -encrypt-with-key sda_key.pub.pem -r data_files_unenc -targetDir "$uploadDir"
+
+check_encrypted_file data_files_unenc/data_file.c4gh data_files_unenc/dir1/data_file1.c4gh
+
+for k in data_files_unenc/data_file.c4gh data_files_unenc/dir1/data_file1.c4gh
+do
+    check_uploaded_file "test/$user/$uploadDir/$k" "$k"
+done
+
+if ! s3cmd -c testing/directS3 ls -r s3://test/"$user"/testEncryptUpload/data_files_unenc/ | grep -v -q 'c4gh'
+then
+    echo "No unencrypted files were uploaded during encrypt+upload"
+else
+    echo "Unencrypted files were uploaded during encrypt+upload"
+    exit 1
+fi
+
+echo "Integration tests for sda-cli upload finished successfully"

.github/integration/tests/30_list.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+if ./sda-cli -config testing/s3cmd.conf list | grep -q 'data_file.c4gh'
+then
+    echo "Listed file from s3 backend"
+else
+    echo "Failed to list file to s3 backend"
+    exit 1
+fi
+
+
+# Check listing files in a dataset
+output=$(./sda-cli -config testing/s3cmd-download.conf list -dataset https://doi.example/ty009.sfrrss/600.45asasga -url http://localhost:8080)
+expected="FileIDSizePathurn:neic:001-0011.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder/dummy_data.c4ghurn:neic:001-0021.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder2/dummy_data2.c4ghurn:neic:001-0031.0MB5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8_elixir-europe.org/main/subfolder2/random/dummy_data3.c4ghDatasetsize:3.1MB"
+if [[ "${output//[$' \t\n\r']/}" == "${expected//[$' \t\n\r']/}" ]];  then
+    echo "Successfully listed files in dataset"
+else
+    echo "Failed to list files in dataset"
+    exit 1
+fi
+
+# Check listing datasets
+output=$(./sda-cli -config testing/s3cmd-download.conf list --datasets -url http://localhost:8080)
+expected="https://doi.example/ty009.sfrrss/600.45asasga"
+if [[ $output == *"$expected"* ]]; then
+    echo "Successfully listed datasets"
+else
+    echo "Failed to list datasets"
+    exit 1
+fi
+
+echo "Integration tests for sda-cli list finished successfully"