Merge branch 'dev' into topic/lint_inputs_missing_from_command

.ci/eslint_wrapper.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+# Setting NODE_PATH and config appropriately, using dependencies from
+# client/node_modules, run eslint against args passed to this script.
+# Primary use case here is for a pre-commit check.
+NODE_PATH=src/ node client/node_modules/eslint/bin/eslint.js -c client/.eslintrc.json "$@"

.ci/first_startup.sh

@@ -4,6 +4,8 @@ case "$1" in
         SCRIPT=./run.sh
         PORT=8080
         LOGFILE=galaxy.log
+        GRAVITY_LOGFILE=database/gravity/log/gunicorn.log
+        SUPERVISORD_LOGFILE=database/gravity/supervisor/supervisord.log
         ;;
     reports)
         SCRIPT=./run_reports.sh
@@ -28,5 +30,16 @@ $SCRIPT --daemon && \
     done
 $SCRIPT --skip-wheels --stop-daemon
 echo "exit code:$EXIT_CODE, showing startup log:"
-cat "$LOGFILE"
+if [ -f "$LOGFILE" ]; then
+    cat "$LOGFILE"
+elif [ ! -f "$LOGFILE" -a -f "$GRAVITY_LOGFILE" ]; then
+    echo "Warning: $LOGFILE does not exist, showing gravity startup log instead"
+    cat "$GRAVITY_LOGFILE"
+elif [ ! -f "$LOGFILE" -a ! -f "$GRAVITY_LOGFILE" -a -f "$SUPERVISORD_LOGFILE" ]; then
+    echo "Warning: $LOGFILE and $GRAVITY_LOGFILE do not exist, showing supervisord startup log instead"
+    cat "$SUPERVISORD_LOGFILE"
+else
+    echo "ERROR: No log files found!"
+    ls -lR database/gravity
+fi
 exit $EXIT_CODE

.ci/flake8_wrapper.sh

@@ -5,4 +5,4 @@ set -e
 flake8 --exclude $(paste -sd, .ci/flake8_ignorelist.txt) .
 
 # Apply stricter rules for the directories shared with Pulsar
-flake8 --ignore=D --max-line-length=150 lib/galaxy/jobs/runners/util/
+flake8 --ignore=E203,D --max-line-length=150 lib/galaxy/jobs/runners/util/

.circleci/config.yml

@@ -16,9 +16,7 @@ variables:
       paths:
         - ~/.cache/yarn
   install_tox: &install_tox
-    run: sudo pip install tox
-  install_deb_reqs: &install_deb_reqs
-    run: sudo apt-get update && sudo apt-get install ffmpeg nodejs -y
+    run: pip install tox
   set_workdir: &set_workdir
     working_directory: ~/repo
   requires_get_code: &requires_get_code
@@ -27,7 +25,7 @@ variables:
 jobs:
   get_code:
     docker:
-      - image: circleci/python:3.7
+      - image: cimg/python:3.7
     <<: *set_workdir
     steps:
       # Replace standard code checkout with shallow clone to speed things up.
@@ -85,40 +83,29 @@ jobs:
             - ~/repo
   validate_test_tools:
     docker:
-      - image: circleci/python:3.7
+      - image: cimg/python:3.7
     <<: *set_workdir
     steps:
       - *restore_repo_cache
       - run: sudo apt-get update
       - run: sudo apt-get install -y libxml2-utils
       - *install_tox
       - run: tox -e validate_test_tools
-  test_galaxy_packages:
-    docker:
-      - image: circleci/python:3.7
-    <<: *set_workdir
-    steps:
-      - *restore_repo_cache
-      - *install_tox
-      - *install_deb_reqs
-      - run: tox -e test_galaxy_packages
   js_lint:
     docker:
-      - image: cimg/node:14.15
+      - image: cimg/node:16.15.1
     <<: *set_workdir
     steps:
       - *restore_yarn_cache
       - run: cd client && yarn install --frozen-lockfile
       - *save_yarn_cache
       - run: cd client && yarn run eslint
-      - run: cd client && yarn run prettier-check
+      - run: cd client && yarn run format-check
 workflows:
   version: 2
   get_code_and_test:
     jobs:
       - get_code
-      - test_galaxy_packages:
-          <<: *requires_get_code
       - validate_test_tools:
           <<: *requires_get_code
       - js_lint:

.flake8

@@ -0,0 +1,10 @@
+[flake8]
+# These are exceptions allowed by Galaxy style guidelines:
+# B008 Do not perform function calls in argument defaults (for FastAPI Depends and Body)
+# E203 is whitespace before ':'; we follow black's formatting here. See https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated
+# E402 module level import not at top of file # TODO, we would like to improve this.
+# E501 is line length
+# W503 is line breaks before binary operators, which has been reversed in PEP 8.
+# D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time).
+ignore = B008,E203,E402,E501,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413
+exclude = lib/galaxy/util/jstree.py

.git-blame-ignore-revs

@@ -0,0 +1,10 @@
+# Migrate code style to Black and Apply isort
+623a5e165ad79e15321b19b2a2afbf2f9224cc90
+# Application of black styling to /tools/
+03351d2a9f81b938d650e2a955c65c18526ee7f0
+# Migrate code style to Prettier
+5b2928f851bd5ea3b9c2a04abf2cee9ff0bc54cc
+87873c5e2f4e6b97fe0f2084bfca0295fcd471de
+# vue/recommended standardization (attribute and property order)
+3bea88e761473b5d67d215d2faf732c86abd2091
+44ac26da4c53dedad3499cfef6d9d4cb61364a05

.github/codecov.yml

@@ -0,0 +1 @@
+comment: false

.github/labeler.yml

@@ -4,8 +4,13 @@ area/admin:
   - doc/source/admin/**/*
 area/API:
   - lib/galaxy/webapps/galaxy/api/**/*
+area/auth:
+  - lib/galaxy/auth/**/*
+  - lib/galaxy/authnz/**/*
 area/client:
   - client/*
+area/database:
+  - lib/galaxy/model/**/*
 area/datatypes:
   - lib/galaxy/datatypes/**/*
   - lib/galaxy/config/sample/datatypes_conf.xml.sample

.github/pull_request_template.md

@@ -10,5 +10,4 @@
   1. [add testing steps and prerequisites here if you didn't write automated tests covering all your changes]
 
 ## License
-- [x] I agree to license these contributions under [Galaxy's current license](https://github.com/galaxyproject/galaxy/blob/dev/LICENSE.txt).
-- [x] I agree to allow the Galaxy committers to license these and all my past contributions to the core galaxy codebase under the [MIT license](https://opensource.org/licenses/MIT). If this condition is an issue, uncheck and just let us know why with an e-mail to [email protected].
+- [x] I agree to license these and all my past contributions to the core galaxy codebase under the [MIT license](https://opensource.org/licenses/MIT).

.github/workflows/api.yaml

@@ -10,21 +10,26 @@ on:
       - 'client/**'
       - 'doc/**'
       - 'lib/galaxy_test/selenium/**'
+  schedule:
+    # Run at midnight UTC every Tuesday
+    - cron: '0 0 * * 2'
 env:
+  GALAXY_DEPENDENCIES_INSTALL_WEASYPRINT: '1'
   GALAXY_TEST_DBURI: 'postgresql://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8'
   GALAXY_TEST_RAISE_EXCEPTION_ON_HISTORYLESS_HDA: '1'
-  GALAXY_DEPENDENCIES_INSTALL_WEASYPRINT: '1'
   GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
 concurrency:
-  group: api-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ['3.7']
+        chunk: [0, 1]
     services:
       postgres:
         image: postgres:13
@@ -35,31 +40,39 @@ jobs:
         ports:
           - 5432:5432
     steps:
-      - uses: actions/checkout@v2
+      - if: github.event_name == 'schedule'
+        run: |
+          echo "GALAXY_CONFIG_OVERRIDE_METADATA_STRATEGY=extended" >> $GITHUB_ENV
+          echo "GALAXY_CONFIG_OVERRIDE_OUTPUTS_TO_WORKING_DIRECTORY=true" >> $GITHUB_ENV
+      - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
       - name: Get full Python version
         id: full-python-version
         shell: bash
         run: echo ::set-output name=version::$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))")
       - name: Cache pip dir
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
       - name: Cache galaxy venv
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
-          path: .venv
+          path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-api
       - name: Run tests
-        run: ./run_tests.sh --skip_flakey_fails -api
+        run: ./run_tests.sh --coverage --skip_flakey_fails -api lib/galaxy_test/api -- --num-shards=2 --shard-id=${{ matrix.chunk }}
         working-directory: 'galaxy root'
-      - uses: actions/upload-artifact@v2
+      - uses: codecov/codecov-action@v2
+        with:
+          flags: api
+          working-directory: 'galaxy root'
+      - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: API test results (${{ matrix.python-version }})
+          name: API test results (${{ matrix.python-version }}, ${{ matrix.chunk }})
           path: 'galaxy root/run_api_tests.html'

.github/workflows/build_container_image.yaml

@@ -5,15 +5,15 @@ on:
       - 'release*'
       - dev
 concurrency:
-  group: docker-build-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
   build:
     name: Build container image
     runs-on: ubuntu-latest
     if: github.repository_owner == 'galaxyproject'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       # https://stackoverflow.com/questions/59810838/how-to-get-the-short-sha-for-the-github-workflow
       - name: Set outputs
         id: commit
@@ -22,9 +22,9 @@ jobs:
         id: branch
         run: echo "::set-output name=name::$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/})"
       - name: Build container image
-        run: docker build . --build-arg GIT_COMMIT=$(git rev-parse HEAD) --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg IMAGE_TAG=${{ steps.branch.outputs.name }} -t galaxyproject/galaxy-min:${{ steps.branch.outputs.name }} -t quay.io/galaxyproject/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile
+        run: docker build . --build-arg GIT_COMMIT=$(git rev-parse HEAD) --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg IMAGE_TAG=${{ steps.branch.outputs.name }} -t galaxy/galaxy-min:${{ steps.branch.outputs.name }} -t quay.io/galaxyproject/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile
       - name: Create auto-expiring one for per-commit auto repository
-        run: echo "FROM galaxyproject/galaxy-min:${{ steps.branch.outputs.name }}" | docker build --label "quay.expires-after"="90d" -t "quay.io/galaxyproject/galaxy-k8s-auto:${{ steps.commit.outputs.sha_short }}" -
+        run: echo "FROM galaxy/galaxy-min:${{ steps.branch.outputs.name }}" | docker build --label "quay.expires-after"="90d" -t "quay.io/galaxyproject/galaxy-k8s-auto:${{ steps.commit.outputs.sha_short }}" -
       - name: Login to quay.io
         uses: actions-hub/docker/login@master
         env:
@@ -47,4 +47,4 @@ jobs:
       - name: Push to DockerHub with branch name
         uses: actions-hub/docker@master
         with:
-          args: push galaxyproject/galaxy-min:${{ steps.branch.outputs.name }}
+          args: push galaxy/galaxy-min:${{ steps.branch.outputs.name }}

.github/workflows/codeql-analysis.yml

@@ -34,37 +34,39 @@ jobs:
       matrix:
         language: ['javascript', 'python']
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
+    #   If the Autobuild fails above, remove it and uncomment the following three lines
+    #   modify them (or add more) to build your code if your project
+    #   uses a compiled language
 
     #- run: |
     #   make bootstrap
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2