.appveyor.yml

# This CI setup provides a largely homogeneous configuration across all
# major platforms (Windows, MacOS, and Linux). The aim of this test setup is
# to create a "native" platform experience, using as few cross-platform
# helper tools as possible.
#
# On Linux/Mac a virtualenv is used for testing. The effective virtual env
# is available under ~/VENV.
#
# All workers support remote login. Login details are shown at the top of each
# CI run log.
#
# - Linux/Mac workers (via SSH):
#
#   - A permitted SSH key must be defined in an APPVEYOR_SSH_KEY environment
#     variable (via the appveyor project settings)
#
#   - SSH login info is given in the form of: 'appveyor@67.225.164.xx -p 22xxx'
#
#   - Login with:
#
#     ssh -o StrictHostKeyChecking=no <LOGIN>
#
#   - to prevent the CI run from exiting, `touch` a file named `BLOCK` in the
#     user HOME directory (current directory directly after login). The session
#     will run until the file is removed (or 60 min have passed)
#
# - Windows workers (via RDP):
#
#   - An RDP password should be defined in an APPVEYOR_RDP_PASSWORD environment
#     variable (via the appveyor project settings), or a random password is used
#     every time
#
#   - RDP login info is given in the form of IP:PORT
#
#   - Login with:
#
#     xfreerdp /cert:ignore /dynamic-resolution /u:appveyor /p:<PASSWORD> /v:<LOGIN>
#
#   - to prevent the CI run from exiting, create a textfile named `BLOCK` on the
#     Desktop (a required .txt extension will be added automatically). The session
#     will run until the file is removed (or 60 min have passed)
#
#   - in a terminal execute, for example, `C:\datalad_debug.bat 39` to set up the
#     environment to debug in a Python 3.8 session (should generally match the
#     respective CI run configuration).


# do not make repository clone cheap: interfers with versioneer
shallow_clone: false

# turn of support for MS project build support (not needed)
build: off

environment:
  # unless indicated otherwise, we test datalad_next
  DTS: datalad_next
  # SSH testing is done via a side-loaded container that provides a POSIX/SSHable
  # server environment
  DATALAD_TESTS_DOCKER_SSHD_SECKEY_DOWNLOADURL: https://ci.appveyor.com/api/projects/mih/datalad-ci-docker-containers/artifacts/recipes/sshd/id_rsa?job=sshd
  DATALAD_TESTS_DOCKER_SSHD_DOWNLOADURL: https://ci.appveyor.com/api/projects/mih/datalad-ci-docker-containers/artifacts/sshd.dockerimg.gz?job=sshd
  DATALAD_TESTS_DOCKER_SSHD_CONTAINER_NAME: test-sshd
  # std SSH container runtime setup
  DATALAD_TESTS_SERVER_SSH_HOST: datalad-test-sshd
  DATALAD_TESTS_SERVER_SSH_PORT: 2222
  DATALAD_TESTS_SERVER_SSH_LOGIN: sshuser
  DATALAD_TESTS_SERVER_SSH_SECKEY: /home/appveyor/.ssh/datalad_tests_id_rsa
  DATALAD_TESTS_SERVER_SSH_PATH: /usr/local/apache2/htdocs
  DATALAD_TESTS_SERVER_LOCALPATH: /home/appveyor/DLTMP/sshdroot

  # Do not use `image` as a matrix dimension, to have fine-grained control over
  # what tests run on which platform
  # The ID variable had no impact, but sorts first in the CI run overview
  # an intelligible name can help to locate a specific test run
  matrix:
    # List a CI run for each platform first, to have immediate access when there
    # is a need for debugging

    # Ubuntu core tests
    - job_name: test-linux
      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
      PY: 3.10
      # datalad-annex git remote needs something after git-annex_8.20211x
      INSTALL_GITANNEX: git-annex -m snapshot
      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
      # 'test_publish_target_url' relies on a strict `localhost` target
      # setup, we don't have that
      KEYWORDS: not test_publish_target_url
      DEPLOY_HTTPBIN_IMAGE: yes
      INSTALL_SYSPKGS:
      DATALAD_TESTS_SSH: 1

    # same as 'test-linux', but TMPDIR is on a crippled filesystem, causing
    # most, if not all test datasets to be created on that filesystem
    - job_name: test-linux-crippled
      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
      PY: 3.10
      # datalad-annex git remote needs something after git-annex_8.20211x
      INSTALL_GITANNEX: git-annex -m snapshot
      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
      DEPLOY_HTTPBIN_IMAGE: yes
      INSTALL_SYSPKGS:
      DATALAD_TESTS_SSH: 1

    # Windows core tests
    - job_name: test-win
      # ~35 min
      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
      # Python version specification is non-standard on windows
      PY: 39-x64
      INSTALL_GITANNEX: git-annex -m datalad/packages
      DATALAD_TESTS_SSH: 1
      DATALAD_TESTS_SERVER_SSH_SECKEY: C:\DLTMP\datalad_tests_id_rsa
      DATALAD_TESTS_SERVER_LOCALPATH: C:\DLTMP\sshdroot

    # MacOS core tests
    - job_name: test-mac
      APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
      PY: 3.8
      INSTALL_GITANNEX: git-annex
      DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
      CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov
      DATALAD_TESTS_SSH: 1
      # no docker on Mac, we log into self
      # 'test_publish_target_url' relies git-annex being installed on the
      # target, but we only have that in a personal env
      KEYWORDS: not test_publish_target_url
      DATALAD_TESTS_SERVER_SSH_HOST: localhost
      DATALAD_TESTS_SERVER_SSH_PORT: 22
      DATALAD_TESTS_SERVER_SSH_LOGIN: appveyor
      DATALAD_TESTS_SERVER_SSH_SECKEY: /Users/appveyor/.ssh/datalad_tests_id_rsa
      DATALAD_TESTS_SERVER_SSH_PATH: /Users/appveyor/DLTMP/riaroot
      DATALAD_TESTS_SERVER_LOCALPATH: /Users/appveyor/DLTMP/riaroot

    # run a subset of the core tests on the oldest supported Python version
    - job_name: datalad-core-1
      DTS: >
        datalad.cli
        datalad.core
      # do not run tests that ensure behavior we intentionally changed
      # - test_gh1811: is included in next in an alternative implementation
      # - test_librarymode: assumes that CLI config overrides end up in the
      #   session `datalad.cfg.overrides`, but -next changes that behavior
      #   to have `.overrides` be uniformly limited to instance overrides
      KEYWORDS: not test_gh1811 and not test_librarymode
      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
      PY: 3.8
      INSTALL_SYSPKGS:
      # datalad-annex git remote needs something after git-annex_8.20211x
      INSTALL_GITANNEX: git-annex -m snapshot
      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
    - job_name: datalad-core-2
      DTS: >
        datalad.customremotes
        datalad.dataset
        datalad.distributed
        datalad.downloaders
        datalad.interface
      # do not run tests that ensure behavior we intentionally changed
      # - test_gh1811: is included in next in an alternative implementation
      # - test_fake_gitlab: we have an updated variant in next
      # - test_dryrun: we have an updated variant in next; what is disabled is
      #   the one in test_create_sibling_gitlab.py. However, there is one with
      #   identical name in test_create_sibling_ghlike.py, now also disabled
      #   because MIH does not know better
      KEYWORDS: >
        not test_gh1811
        and not test_nested_pushclone_cycle_allplatforms
        and not test_fake_gitlab
        and not test_dryrun
        and not test_initremote
        and not test_push_url
        and not test_read_access
        and not test_ria_postclone_noannex
        and not test_version_check_ssh
        and not test_version_check
        and not test_obtain_permission_ssh
        and not test_obtain_permission_root
        and not test_remote_layout
        and not test_remote_layout_ssh
        and not test_setup_store
        and not test_setup_ds_in_store
        and not test_initremote_basic_fileurl
        and not test_initremote_basic_httpurl
        and not test_initremote_basic_httpsurl
        and not test_initremote_basic_sshurl
        and not test_gitannex_local
        and not test_gitannex_ssh
        and not test_ria_postclonecfg
      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
      PY: 3.8
      INSTALL_SYSPKGS:
      # datalad-annex git remote needs something after git-annex_8.20211x
      INSTALL_GITANNEX: git-annex -m snapshot
      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
    - job_name: datalad-core-3
      DTS: >
        datalad.distribution
      KEYWORDS: not test_invalid_args
      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
      PY: 3.8
      INSTALL_SYSPKGS:
      # datalad-annex git remote needs something after git-annex_8.20211x
      INSTALL_GITANNEX: git-annex -m snapshot
      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
    - job_name: datalad-core-4
      DTS: >
        datalad.local
      KEYWORDS: >
        not test_wtf
      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
      PY: 3.8
      INSTALL_SYSPKGS:
      # datalad-annex git remote needs something after git-annex_8.20211x
      INSTALL_GITANNEX: git-annex -m snapshot
      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
    - job_name: datalad-core-5
      DTS: >
        datalad.runner
        datalad.support
        datalad.tests
        datalad.ui
      APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
      PY: 3.8
      INSTALL_SYSPKGS:
      # datalad-annex git remote needs something after git-annex_8.20211x
      INSTALL_GITANNEX: git-annex -m snapshot
      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov


# only run the CI if there are code or tooling changes
only_commits:
  files:
    - datalad_next/
    - tools/


# tests need specific hostnames to be available
# note, this is insufficient on MacOS, and needs to be reflected
# in the SSH config too
hosts:
  datalad-test-sshd: 127.0.0.1
  # same, but for datalad-core implementations
  datalad-test: 127.0.0.1


# job-specific configurations
for:
  #
  # POSIX TEST RUNS
  #
  - matrix:
      only:
        - job_name: test-linux
        - job_name: test-linux-crippled
        - job_name: test-mac
        - job_name: datalad-core-1
        - job_name: datalad-core-2
        - job_name: datalad-core-3
        - job_name: datalad-core-4
        - job_name: datalad-core-5

    cache:
      # pip cache
      - /home/appveyor/.cache/pip -> .appveyor.yml
      # cache the docker image for httpbin. in 2023 it has not changed in
      # 4 years, not worth pulling each time
      # given the low change frequency we also do not invalidate the cache
      # but would do manually, if needed
      - /home/appveyor/cache/httpbin.dockerimg

    # init cannot use any components from the repo, because it runs prior to
    # cloning it
    init:
      # enable external SSH access to CI worker
      # needs APPVEYOR_SSH_KEY defined in project settings (or environment)
      - curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
      # Scratch space
      # we place the "unix" one into the user's HOME to avoid git-annex issues on MacOSX
      # gh-5291
      - mkdir ~/DLTMP && export TMPDIR=~/DLTMP
      # establish a root directory for SSH tests
      - "mkdir ${DATALAD_TESTS_SERVER_LOCALPATH}"

    install:
      # verify that a PY variable is declared that identifies the desired Python version
      # for this run
      - "[ \"x$PY\" != x ]"
      # Missing system software
      - tools/appveyor/install-syspkgs $INSTALL_SYSPKGS
      # If a particular Python version is requested, use env setup (using the
      # appveyor provided environments/installation). Note, these are broken
      # on the ubuntu images
      # https://help.appveyor.com/discussions/problems/28217-appveyor-ubunu-image-with-python3-lzma-module
      # Otherwise create a virtualenv using the default Python 3, to enable uniform
      # use of python/pip executables below
      - "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || virtualenv -p 3 ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\""
      - tools/appveyor/install-git-annex ${INSTALL_GITANNEX}
      # enable the git-annex provisioned by the installer
      - "[ -f ${HOME}/dlinstaller_env.sh ] && . ${HOME}/dlinstaller_env.sh || true"
      # HTTPBIN
      - "[ -n \"$DEPLOY_HTTPBIN_IMAGE\" ] && tools/appveyor/docker-load-httpbin || true"

    before_test:
      - "[ \"$DATALAD_TESTS_SSH\" = 1 ] && tools/appveyor/setup-sshd || true"
      - "[ \"$DATALAD_TESTS_SSH\" = 1 ] && tools/appveyor/verify-ssh-access || true"

    test_script:
      # store original TMPDIR setting to limit modification to test execution
      - export PREV_TMPDIR=$TMPDIR
      # make TMPDIR a "crippled filesystem" to test wrong assumptions of POSIX-ness
      # on POSIX OSes. The test fixtures will create all test datasets under TMPDIR
      - |
        set -e
        if [ "$APPVEYOR_JOB_NAME" = "test-linux-crippled" ]; then
          # 750 MB VFAT FS in a box
          sudo dd if=/dev/zero of=/crippledfs.img count=750 bs=1M
          sudo mkfs.vfat /crippledfs.img
          sudo mkdir /crippledfs
          sudo mount -o "uid=$(id -u),gid=$(id -g)" /crippledfs.img /crippledfs
          echo "== mount >>"
          mount | grep crippled
          echo "<< mount =="
          export TMPDIR=/crippledfs
        fi
      - echo TMPDIR=$TMPDIR
      # run tests on installed module, not source tree files
      - mkdir __testhome__
      # run tests on installed module, not source tree files
      - |
        cd __testhome__
        python -m pytest -s -v --durations 20 -m "not (turtle)" -k "$KEYWORDS" --cov=datalad_next --cov datalad --cov-config=../.coveragerc --pyargs ${DTS}

    after_test:
      - python -m coverage xml
      - "curl -Os $CODECOV_BINARY"
      - chmod +x codecov
      - ./codecov

    on_finish:
      # conditionally block the exit of a CI run for direct debugging
      - while [ -f ~/BLOCK ]; do sleep 5; done


  #
  # WINDOWS TEST RUNS
  #
  - matrix:
      only:
        - job_name: test-win
    cache:
      # pip cache
      - C:\Users\appveyor\AppData\Local\pip\Cache -> .appveyor.yml

    # init cannot use any components from the repo, because it runs prior to
    # cloning it
    init:
      # remove windows 260-char limit on path names
      - ps: Set-Itemproperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name LongPathsEnabled -value 1
      # enable developer mode on windows
      # this should enable mklink without admin privileges, but it doesn't seem to work
      #- ps: tools\ci\appveyor_enable_windevmode.ps1
      # enable RDP access on windows (RDP password is in appveyor project config)
      # this is relatively expensive (1-2min), but very convenient to jump into any build at any time
      - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
      # Scratch space
      - cmd: md C:\DLTMP
      # and use that scratch space to get short paths in test repos
      # (avoiding length-limits as much as possible)
      - cmd: "set TMP=C:\\DLTMP"
      - cmd: "set TEMP=C:\\DLTMP"
      # establish a root directory for SSH tests
      - cmd: "md %DATALAD_TESTS_SERVER_LOCALPATH%"

    install:
      # place a debug setup helper at a convenient location
      - cmd: copy tools\appveyor\env_setup.bat C:\\datalad_debug.bat
      - cmd: "set PATH=C:\\Python%PY%;C:\\Python%PY%\\Scripts;%PATH%"
      # deploy the datalad installer, override version via DATALAD_INSTALLER_VERSION
      - cmd:
          IF DEFINED DATALAD_INSTALLER_VERSION (
          python -m pip install "datalad-installer%DATALAD_INSTALLER_VERSION%"
          ) ELSE (
          python -m pip install datalad-installer
          )
      # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used
      # deploy git-annex, if desired
      - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX%

    before_test:
      - tools\appveyor\setup-sshd
      - tools\appveyor\verify-ssh-access

    test_script:
      # run tests on installed module, not source tree files
      - cmd: md __testhome__
      - cmd: cd __testhome__
        # run test selection
      - cmd: python -m pytest -s -v --durations 20 -m "not (turtle)" -k "%KEYWORDS%" --cov=datalad_next --cov-config=..\.coveragerc --pyargs %DTS%

    after_test:
      - cmd: python -m coverage xml
      - cmd: curl -fsSL -o codecov.exe "https://uploader.codecov.io/latest/windows/codecov.exe"
      - cmd: .\codecov.exe -f "coverage.xml"

    on_finish:
      # conditionally block the exit of a CI run for direct debugging
      - ps: while ((Test-Path "C:\Users\\appveyor\\Desktop\\BLOCK.txt")) { Start-Sleep 5 }


#
# ALL TEST RUNS
#
build_script:
  - python -m pip install -r requirements-devel.txt
  - python -m pip install .

after_build:
  # Identity setup
  - git config --global user.email "test@appveyor.land"
  - git config --global user.name "Appveyor Almighty"
  # enable "next" extension for patching datalad core
  - git config --global datalad.extensions.load next
  # simple call to see if datalad and git-annex are installed properly
  - datalad wtf