Skip to content

Commit

Permalink
SSHTMP
Browse files Browse the repository at this point in the history
  • Loading branch information
mih committed Dec 13, 2023
1 parent 69db927 commit 8863580
Show file tree
Hide file tree
Showing 3 changed files with 183 additions and 98 deletions.
219 changes: 121 additions & 98 deletions .appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,13 @@ shallow_clone: false
build: off

environment:
# we do not have an adequate setup for SSH-based tests right now
#DATALAD_TESTS_SSH: 1
# unless indicated otherwise, we test datalad_next
DTS: datalad_next
# SSH testing is done via a side-loaded container that provides a POSIX/SSHable
# server environment
DATALAD_TESTS_DOCKER_SSHD_SECKEY_DOWNLOADURL: https://ci.appveyor.com/api/projects/mih/datalad-ci-docker-containers/artifacts/recipes/sshd/id_rsa?job=sshd
DATALAD_TESTS_DOCKER_SSHD_DOWNLOADURL: https://ci.appveyor.com/api/projects/mih/datalad-ci-docker-containers/artifacts/sshd.dockerimg.gz?job=sshd
DATALAD_TESTS_DOCKER_SSHD_CONTAINER_NAME: test-sshd

# Do not use `image` as a matrix dimension, to have fine-grained control over
# what tests run on which platform
Expand All @@ -74,103 +77,110 @@ environment:
CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
DEPLOY_HTTPBIN_IMAGE: yes
INSTALL_SYSPKGS:
#DATALAD_TESTS_SSH: 1
DATALAD_TESTS_SERVER_SSH_HOST: datalad-test-sshd
DATALAD_TESTS_SERVER_SSH_PORT: 2222
DATALAD_TESTS_SERVER_SSH_LOGIN: sshuser
DATALAD_TESTS_SERVER_SSH_SECKEY: /home/appveyor/.ssh/datalad_tests_id_rsa
DATALAD_TESTS_SERVER_SSH_PATH: /usr/local/apache2/htdocs
DATALAD_TESTS_SERVER_LOCALPATH: /home/appveyor/DLTMP/sshdroot

# same as 'test-linux', but TMPDIR is on a crippled filesystem, causing
# most, if not all test datasets to be created on that filesystem
- job_name: test-linux-crippled
APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
PY: 3.10
# datalad-annex git remote needs something after git-annex_8.20211x
INSTALL_GITANNEX: git-annex -m snapshot
CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
DEPLOY_HTTPBIN_IMAGE: yes
INSTALL_SYSPKGS:

# Windows core tests
- job_name: test-win
# ~35 min
APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
# Python version specification is non-standard on windows
PY: 39-x64
INSTALL_GITANNEX: git-annex -m datalad/packages

# MacOS core tests
- job_name: test-mac
APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
PY: 3.8
INSTALL_GITANNEX: git-annex
DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov

# run a subset of the core tests on the oldest supported Python version
- job_name: datalad-core-1
DTS: >
datalad.cli
datalad.core
# do not run tests that ensure behavior we intentionally changed
# - test_gh1811: is included in next in an alternative implementation
# - test_librarymode: assumes that CLI config overrides end up in the
# session `datalad.cfg.overrides`, but -next changes that behavior
# to have `.overrides` be uniformly limited to instance overrides
KEYWORDS: not test_gh1811 and not test_librarymode
APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
PY: 3.8
INSTALL_SYSPKGS:
# datalad-annex git remote needs something after git-annex_8.20211x
INSTALL_GITANNEX: git-annex -m snapshot
CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
- job_name: datalad-core-2
DTS: >
datalad.customremotes
datalad.dataset
datalad.distributed
datalad.downloaders
datalad.interface
# do not run tests that ensure behavior we intentionally changed
# - test_gh1811: is included in next in an alternative implementation
# - test_fake_gitlab: we have an updated variant in next
# - test_dryrun: we have an updated variant in next; what is disabled is
# the one in test_create_sibling_gitlab.py. However, there is one with
# identical name in test_create_sibling_ghlike.py, now also disabled
# because MIH does not know better
KEYWORDS: not test_gh1811 and not test_fake_gitlab and not test_dryrun
APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
PY: 3.8
INSTALL_SYSPKGS:
# datalad-annex git remote needs something after git-annex_8.20211x
INSTALL_GITANNEX: git-annex -m snapshot
CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
- job_name: datalad-core-3
DTS: >
datalad.distribution
KEYWORDS: not test_invalid_args
APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
PY: 3.8
INSTALL_SYSPKGS:
# datalad-annex git remote needs something after git-annex_8.20211x
INSTALL_GITANNEX: git-annex -m snapshot
CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
- job_name: datalad-core-4
DTS: >
datalad.local
APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
PY: 3.8
INSTALL_SYSPKGS:
# datalad-annex git remote needs something after git-annex_8.20211x
INSTALL_GITANNEX: git-annex -m snapshot
CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
- job_name: datalad-core-5
DTS: >
datalad.runner
datalad.support
datalad.tests
datalad.ui
APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
PY: 3.8
INSTALL_SYSPKGS:
# datalad-annex git remote needs something after git-annex_8.20211x
INSTALL_GITANNEX: git-annex -m snapshot
CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
# # same as 'test-linux', but TMPDIR is on a crippled filesystem, causing
# # most, if not all test datasets to be created on that filesystem
# - job_name: test-linux-crippled
# APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
# PY: 3.10
# # datalad-annex git remote needs something after git-annex_8.20211x
# INSTALL_GITANNEX: git-annex -m snapshot
# CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
# DEPLOY_HTTPBIN_IMAGE: yes
# INSTALL_SYSPKGS:
#
# # Windows core tests
# - job_name: test-win
# # ~35 min
# APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
# # Python version specification is non-standard on windows
# PY: 39-x64
# INSTALL_GITANNEX: git-annex -m datalad/packages
#
# # MacOS core tests
# - job_name: test-mac
# APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
# PY: 3.8
# INSTALL_GITANNEX: git-annex
# DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
# CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov
#
# # run a subset of the core tests on the oldest supported Python version
# - job_name: datalad-core-1
# DTS: >
# datalad.cli
# datalad.core
# # do not run tests that ensure behavior we intentionally changed
# # - test_gh1811: is included in next in an alternative implementation
# # - test_librarymode: assumes that CLI config overrides end up in the
# # session `datalad.cfg.overrides`, but -next changes that behavior
# # to have `.overrides` be uniformly limited to instance overrides
# KEYWORDS: not test_gh1811 and not test_librarymode
# APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
# PY: 3.8
# INSTALL_SYSPKGS:
# # datalad-annex git remote needs something after git-annex_8.20211x
# INSTALL_GITANNEX: git-annex -m snapshot
# CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
# - job_name: datalad-core-2
# DTS: >
# datalad.customremotes
# datalad.dataset
# datalad.distributed
# datalad.downloaders
# datalad.interface
# # do not run tests that ensure behavior we intentionally changed
# # - test_gh1811: is included in next in an alternative implementation
# # - test_fake_gitlab: we have an updated variant in next
# # - test_dryrun: we have an updated variant in next; what is disabled is
# # the one in test_create_sibling_gitlab.py. However, there is one with
# # identical name in test_create_sibling_ghlike.py, now also disabled
# # because MIH does not know better
# KEYWORDS: not test_gh1811 and not test_fake_gitlab and not test_dryrun
# APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
# PY: 3.8
# INSTALL_SYSPKGS:
# # datalad-annex git remote needs something after git-annex_8.20211x
# INSTALL_GITANNEX: git-annex -m snapshot
# CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
# - job_name: datalad-core-3
# DTS: >
# datalad.distribution
# KEYWORDS: not test_invalid_args
# APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
# PY: 3.8
# INSTALL_SYSPKGS:
# # datalad-annex git remote needs something after git-annex_8.20211x
# INSTALL_GITANNEX: git-annex -m snapshot
# CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
# - job_name: datalad-core-4
# DTS: >
# datalad.local
# APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
# PY: 3.8
# INSTALL_SYSPKGS:
# # datalad-annex git remote needs something after git-annex_8.20211x
# INSTALL_GITANNEX: git-annex -m snapshot
# CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
# - job_name: datalad-core-5
# DTS: >
# datalad.runner
# datalad.support
# datalad.tests
# datalad.ui
# APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
# PY: 3.8
# INSTALL_SYSPKGS:
# # datalad-annex git remote needs something after git-annex_8.20211x
# INSTALL_GITANNEX: git-annex -m snapshot
# CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov


# do not run the CI if only documentation changes were made
Expand All @@ -189,6 +199,13 @@ skip_commits:
- '*.md'


# tests need specific hostnames to be available
# note, this is insufficient on MacOS, and needs to be reflected
# in the SSH config too
hosts:
datalad-test-sshd: 127.0.0.1


# job-specific configurations
for:
#
Expand Down Expand Up @@ -224,6 +241,8 @@ for:
# we place the "unix" one into the user's HOME to avoid git-annex issues on MacOSX
# gh-5291
- mkdir ~/DLTMP && export TMPDIR=~/DLTMP
# establish a root directory for SSH tests
- "mkdir ${DATALAD_TESTS_SERVER_LOCALPATH}"

install:
# verify that a PY variable is declared that identifies the desired Python version
Expand All @@ -244,6 +263,10 @@ for:
# HTTPBIN
- "[ -n \"$DEPLOY_HTTPBIN_IMAGE\" ] && tools/appveyor/docker-load-httpbin || true"

before_test:
- tools/appveyor/setup-sshd
- tools/appveyor/verify-ssh-access

test_script:
# store original TMPDIR setting to limit modification to test execution
- export PREV_TMPDIR=$TMPDIR
Expand Down
48 changes: 48 additions & 0 deletions tools/appveyor/setup-sshd
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/bin/bash

set -e -u

DATALAD_TESTS_SERVER_SSH_SECKEY=${DATALAD_TESTS_SERVER_SSH_SECKEY:-$HOME/.ssh/id_rsa}


function setup_docker () {
# obtain the docker image for SSH testing
curl -fsSL --ssl-no-revoke -o sshd.dockerimg.gz "${DATALAD_TESTS_DOCKER_SSHD_DOWNLOADURL}"
gzip -c -d sshd.dockerimg.gz | docker load

# obtain the matching SSH private key for SSH server login
curl \
-fsSL \
-o "${DATALAD_TESTS_SERVER_SSH_SECKEY}" \
"${DATALAD_TESTS_DOCKER_SSHD_SECKEY_DOWNLOADURL}"

# start docker container
docker run \
--rm -dit \
--name "${DATALAD_TESTS_DOCKER_SSHD_CONTAINER_NAME}" \
-p "${DATALAD_TESTS_SERVER_SSH_PORT}:22" \
-v "${DATALAD_TESTS_SERVER_LOCALPATH}:${DATALAD_TESTS_SERVER_SSH_PATH}" \
sshd
}

function setup_ssh_localhost () {
ssh-keygen -f "${DATALAD_TESTS_SERVER_SSH_SECKEY}" -N ''
cat "${DATALAD_TESTS_SERVER_SSH_SECKEY}.pub" >> ${HOME}/.ssh/authorized_keys
}

# if there is docker use it, if not, use the worker itself
docker -v && setup_docker || setup_ssh_localhost

# wipe any other known host keys
ssh-keygen \
-f "${HOME}/.ssh/known_hosts" \
-R "[${DATALAD_TESTS_SERVER_SSH_HOST}]:${DATALAD_TESTS_SERVER_SSH_PORT}"

# establish expected permission setup for SSH key
chmod 600 "${DATALAD_TESTS_SERVER_SSH_SECKEY}"

# ingest actual host key
ssh-keyscan \
-t ecdsa \
-p "${DATALAD_TESTS_SERVER_SSH_PORT}" \
-H "${DATALAD_TESTS_SERVER_SSH_HOST}" >> "${HOME}/.ssh/known_hosts"
14 changes: 14 additions & 0 deletions tools/appveyor/verify-ssh-access
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/bin/bash

set -e -u


touch ${DATALAD_TESTS_SERVER_LOCALPATH}/probe

ssh \
-i "${DATALAD_TESTS_SERVER_SSH_SECKEY}" \
-p "${DATALAD_TESTS_SERVER_SSH_PORT}" \
"${DATALAD_TESTS_SERVER_SSH_LOGIN}@${DATALAD_TESTS_SERVER_SSH_HOST}" \
test -f ${DATALAD_TESTS_SERVER_SSH_PATH}/probe

rm ${DATALAD_TESTS_SERVER_LOCALPATH}/probe

0 comments on commit 8863580

Please sign in to comment.