diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index ad7af9083..dde423557 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -22,11 +22,38 @@ jobs: egress-policy: block allowed-endpoints: > api.deps.dev:443 + api.electricitymap.org:443 api.github.com:443 + api.green-coding.io:443 + api.securityscorecards.dev:443 github.com:443 + ip-api.com:80 + ipapi.co:443 + proxy.golang.org:443 + pypi.org:443 + sum.golang.org:443 + + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Dependency Review uses: actions/dependency-review-action@4081bf99e2866ebe428fc0477b69eb4fcda7220a # v4.4.0 + + - name: Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Dependency Review' + continue-on-error: true + + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8403a6450..7c2cb8e54 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,9 +45,22 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 + api.github.com:443 + api.green-coding.io:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 + proxy.golang.org:443 pypi.org:443 + sum.golang.org:443 + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python3 @@ -58,6 +71,12 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Setup Python Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (Python${{ matrix.python-version }}, lint)' + continue-on-error: true - name: Environment Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -71,6 +90,17 @@ jobs: - name: Run linting suite run: | python -m tox -e lint + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Lint (Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true test-preliminary: name: Python${{ matrix.python-version }} (${{ matrix.os }}) @@ -88,12 +118,24 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 - dap.service.does.not.exist:443 + api.green-coding.io:443 + coveralls.io:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 + proxy.golang.org:443 pypi.org:443 raw.githubusercontent.com:443 + sum.golang.org:443 + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Set up Python${{ matrix.python-version }} @@ -104,6 +146,12 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Environment Setup Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (Python${{ matrix.python-version }}, standard)' + continue-on-error: true - name: Environment Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -116,7 +164,19 @@ jobs: python -m tox -- -m 'not slow' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: run-${{ matrix.python-version }}-ubuntu-latest-preliminary COVERALLS_PARALLEL: true + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing with tox (Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true test-pypi: needs: lint @@ -180,18 +240,27 @@ jobs: with: egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 + api.green-coding.io:443 azure.archive.ubuntu.com:80 coveralls.io:443 - dap.service.does.not.exist:443 - esm.ubuntu.com:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 motd.ubuntu.com:443 - packages.microsoft.com:443 - ppa.launchpadcontent.net:443 + proxy.golang.org:443 pypi.org:443 raw.githubusercontent.com:443 + sum.golang.org:443 + - name: Start Energy Measurement + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Eigen3 (SBCK) @@ -207,6 +276,13 @@ jobs: - name: Install CI libraries run: | python -m pip install --require-hashes -r CI/requirements_ci.txt + - name: Environment Setup Energy Measurement + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (Python${{ matrix.python-version }}, ${{ matrix.tox-env }})' + continue-on-error: true - name: Environment Caching # if prefetch is not in tox-env if: contains(matrix.tox-env, 'prefetch') == false @@ -232,6 +308,19 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.tox-env }} COVERALLS_PARALLEL: true + - name: Tests measurement + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing with tox (Python${{ matrix.python-version }}, ${{ matrix.tox-env }})' + continue-on-error: true + - name: Show Energy Results + if : ${{ matrix.os == 'ubuntu-latest' }} + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true test-conda: needs: lint @@ -240,7 +329,7 @@ jobs: contains(github.event.pull_request.labels.*.name, 'approved') || (github.event.review.state == 'approved') || (github.event_name == 'push') - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: matrix: os: [ 'ubuntu-latest' ] @@ -256,16 +345,25 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 - conda.anaconda.org:443 + api.green-coding.io:443 coveralls.io:443 - dap.service.does.not.exist:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 objects.githubusercontent.com:443 + proxy.golang.org:443 pypi.org:443 raw.githubusercontent.com:443 - repo.anaconda.com:443 + sum.golang.org:443 + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} @@ -276,9 +374,6 @@ jobs: environment-file: environment.yml create-args: >- python=${{ matrix.python-version }} - - name: Micromamba version - run: | - echo "micromamba: $(micromamba --version)" - name: Test Data Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -293,21 +388,15 @@ jobs: micromamba list xclim show_version_info python -m pip check || true + - name: Environment Setup Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment setup (conda, Python${{ matrix.python-version }})' + continue-on-error: true - name: Test with pytest run: | python -m pytest --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing -# - name: Install tox -# shell: bash -l {0} -# run: | -# mamba install -n xclim39 tox tox-conda -# - name: Test -# shell: bash -l {0} -# run: | -# conda activate xclim39 -# tox -e opt-slow -# env: -# CONDA_EXE: mamba -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Report coverage run: | coveralls @@ -315,20 +404,64 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_FLAG_NAME: run-{{ matrix.python-version }}-conda COVERALLS_PARALLEL: true + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing with pytest (conda, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true finish: + name: Finish needs: - test-preliminary - test-pypi - test-conda runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - name: Harden Runner uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 with: disable-sudo: true - egress-policy: audit + egress-policy: block + allowed-endpoints: > + api.electricitymap.org:443 + api.github.com:443 + api.green-coding.io:443 + coveralls.io:443 + files.pythonhosted.org:443 + github.com:443 + ip-api.com:80 + ipapi.co:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + pypi.org:443 + sum.golang.org:443 + - name: Start Energy Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + continue-on-error: true - name: Coveralls Finished uses: coverallsapp/github-action@cfd0633edbd2411b532b808ba7a8b5e04f76d2c8 # v2.3.4 with: parallel-finished: true + - name: Finish measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Finish Run' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 7965189a8..8477d39b1 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -35,12 +35,12 @@ jobs: egress-policy: audit - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Run Analysis - uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 with: results_file: results.sarif results_format: sarif @@ -59,7 +59,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: Upload Artifact - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: SARIF file path: results.sarif @@ -67,6 +67,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # 3.23.0 + uses: github/codeql-action/upload-sarif@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # 3.23.0 with: sarif_file: results.sarif diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index a95f774f0..2ea9c0509 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -44,15 +44,27 @@ jobs: disable-sudo: true egress-policy: block allowed-endpoints: > + api.electricitymap.org:443 api.github.com:443 + api.green-coding.io:443 + api.securityscorecards.dev:443 conda.anaconda.org:443 dap.service.does.not.exist:443 files.pythonhosted.org:443 github.com:443 + ip-api.com:80 + ipapi.co:443 objects.githubusercontent.com:443 + proxy.golang.org:4433 pypi.org:443 raw.githubusercontent.com:443 repo.anaconda.com:443 + sum.golang.org:443 + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -84,6 +96,12 @@ jobs: micromamba list xclim show_version_info python -m pip check || true + - name: Setup Python Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Environment Setup (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true - name: Test Data Caching uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: @@ -105,3 +123,14 @@ jobs: with: issue-title: "⚠️ Nightly upstream-dev CI failed for Python${{ matrix.python-version }} ⚠️" log-path: output-${{ matrix.python-version }}-log.jsonl + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: get-measurement + label: 'Testing and Reporting (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@86f1b2ee12db687bca0d15160a529bb64a7b60d9 # v4.0.0 + with: + task: display-results + continue-on-error: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 231c6116c..e2f0c9360 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,17 +4,32 @@ Changelog v0.54.0 (unreleased) -------------------- -Contributors to this version: Trevor James Smith (:user:`Zeitsperre`). +Contributors to this version: Trevor James Smith (:user:`Zeitsperre`), Pascal Bourgault (:user:`aulemahal`), Éric Dupuis (:user:`coxipi`). Breaking changes ----------------- -* The minimum required version of `dask` has been increased to `2024.8.1`. `dask` versions at or above `2024.11` are not yet supported. (:issue:`1992`, :pull:`1991`). +^^^^^^^^^^^^^^^^ +* The minimum required version of `dask` has been increased to `2024.8.1`. (:issue:`1992`, :pull:`1991`). + +Bug fixes +^^^^^^^^^ +* Fixed pickling issue with ``xclim.sdba.Grouper`` and other classes for usage with `dask>=2024.11`. (:issue:`1992`, :pull:`1993`). +* Fixed an issue with ``nimbus`` that was causing URL path components to be improperly joined. (:pull:`1997`). +* `base_kws_vars` in `MBCn` is now copied inside the `adjust` function so that in-place changes do not change the dict globally. (:pull:`1999`). +* Fixed a bug in the logic of ``xclim.testing.utils.load_registry`` that impacted the ability to load a `registry.txt` from a non-default repository. (:pull:`2001`). Internal changes ^^^^^^^^^^^^^^^^ +* Changed french translations with word "pluvieux" to "avec précipitations". (:issue:`1960`, :pull:`1994`). +* `streamflow` entry replaced with `q` in ``variables.yml``. (:issue:`1912`, :pull:`1996`) +* In order to address 403 (forbidden) request errors when retrieving data from GitHub via ReadTheDocs, the ``nimbus`` class has been modified to use an overloaded `fetch` method that appends a User-Agent header to the request. (:pull:`2001`). +* Addressed a very rare race condition that can happen if `pytest` is tearing down the test environment when running across multiple workers. (:pull:`1863`). * `xclim` now uses a `src` layout for the codebase. Structure-dependent functions, documentation, and build commands have been adapted to reflect these changes. (:pull:`1971`). * Added a more robust `yamllint` configuration to ensure that all YAML files are linted consistently. (:pull:`1971`). +CI changes +^^^^^^^^^^ +* Added the `green-coding-solutions/eco-ci-energy-estimation` GitHub Action to the workflows to establish energy and carbon usage of CI activity. (:pull:`1863`). + v0.53.2 (2024-10-31) -------------------- Contributors to this version: Éric Dupuis (:user:`coxipi`), Pascal Bourgault (:user:`aulemahal`), Trevor James Smith (:user:`Zeitsperre`). diff --git a/CI/requirements_ci.in b/CI/requirements_ci.in index dbff75495..d4c6ec4ab 100644 --- a/CI/requirements_ci.in +++ b/CI/requirements_ci.in @@ -1,5 +1,5 @@ bump-my-version==0.28.1 -deptry==0.20.0 +deptry==0.21.1 flit==3.10.1 pip==24.3.1 pylint==3.3.1 diff --git a/CI/requirements_ci.txt b/CI/requirements_ci.txt index 14c8334ea..554869500 100644 --- a/CI/requirements_ci.txt +++ b/CI/requirements_ci.txt @@ -195,9 +195,9 @@ mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba # via markdown-it-py -packaging==24.1 \ - --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ - --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 +packaging==24.2 \ + --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ + --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f # via # pyproject-api # tox diff --git a/README.rst b/README.rst index 05d940f5a..f02d9dbff 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ xclim: Climate services library |logo| |logo-dark| |logo-light| +----------------------------+-----------------------------------------------------+ | Coding Standards | |black| |ruff| |pre-commit| |security| |fossa| | +----------------------------+-----------------------------------------------------+ -| Development Status | |status| |build| |coveralls| | +| Development Status | |status| |build| |coveralls| |energy| | +----------------------------+-----------------------------------------------------+ `xclim` is an operational Python library for climate services, providing numerous climate-related indicator tools @@ -168,6 +168,10 @@ This package was created with Cookiecutter_ and the `audreyfeldroy/cookiecutter- :target: https://bestpractices.coreinfrastructure.org/projects/6041 :alt: Open Source Security Foundation +.. |energy| image:: https://api.green-coding.io/v1/ci/badge/get?repo=Ouranosinc/xclim&branch=main&workflow=47013755 + :target: https://metrics.green-coding.io/ci.html?repo=Ouranosinc/xclim&branch=main&workflow=47013755 + :alt: Energy Consumption + .. |fair| image:: https://img.shields.io/badge/fair--software.eu-%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8B-yellow :target: https://fair-software.eu :alt: FAIR Software Compliance diff --git a/environment.yml b/environment.yml index 4e1333aa2..4d17e8f48 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ dependencies: - cf_xarray >=0.9.3 - cftime >=1.4.1 - click >=8.1 - - dask >=2024.8.1,<2024.11.0 + - dask >=2024.8.1 - filelock >=3.14.0 - jsonpickle >=3.1.0 - numba >=0.54.1 diff --git a/pyproject.toml b/pyproject.toml index 7e59e6ea5..c7e9fe036 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "cf-xarray >=0.9.3", # cf-xarray is differently named on conda-forge "cftime >=1.4.1", "click >=8.1", - "dask[array] >=2024.8.1,<2024.11.0", + "dask[array] >=2024.8.1", "filelock >=3.14.0", "jsonpickle >=3.1.0", "numba >=0.54.1", @@ -65,7 +65,7 @@ dev = [ "codespell ==2.3.0", "coverage[toml] >=7.5.0", "coveralls >=4.0.1", # coveralls is not yet compatible with Python 3.13 - "deptry ==0.20.0", + "deptry ==0.21.1", "flake8 >=7.1.1", "flake8-rst-docstrings >=0.3.0", "h5netcdf>=1.3.0", @@ -135,7 +135,7 @@ target-version = [ ] [tool.bumpversion] -current_version = "0.53.2" +current_version = "0.53.3-dev.6" commit = true commit_args = "--no-verify" tag = false diff --git a/src/xclim/__init__.py b/src/xclim/__init__.py index 2680845d2..c4c1de29a 100644 --- a/src/xclim/__init__.py +++ b/src/xclim/__init__.py @@ -13,7 +13,7 @@ __author__ = """Travis Logan""" __email__ = "logan.travis@ouranos.ca" -__version__ = "0.53.2" +__version__ = "0.53.3-dev.6" with _resources.as_file(_resources.files("xclim.data")) as _module_data: diff --git a/src/xclim/data/fr.json b/src/xclim/data/fr.json index 94822e704..d42c6b437 100644 --- a/src/xclim/data/fr.json +++ b/src/xclim/data/fr.json @@ -206,7 +206,7 @@ "WETDAYS": { "long_name": "Nombre de jours où la précipitation est supérieure ou égale à {thresh}", "description": "Nombre {freq:m} de jours où la précipitation est supérieure ou égale à {thresh}.", - "title": "Jours pluvieux", + "title": "Jours avec précipitations", "abstract": "Nombre de jours où la précipitation est supérieure ou égale à un seuil donné." }, "WETDAYS_PROP": { @@ -224,7 +224,7 @@ "CWD": { "long_name": "Durée maximale d'une période où les précipitations sont au-dessus de {thresh}", "description": "Nombre {freq:m} maximal de jours consécutifs où les précipitations sont au-dessus de {thresh}.", - "title": "Durée de période pluvieuse", + "title": "Durée de la période de précipitation", "abstract": "Période la plus longue où la précipitation est au-dessus d'un seuil donné." }, "SDII": { @@ -1300,13 +1300,13 @@ "WARM_AND_WET_DAYS": { "long_name": "Nombre de jours où la température est au-dessus du {tas_per_thresh}e centile et la précipitation au-dessus du {pr_per_thresh}e centile", "description": "Nombre {freq:m} de jours où la température est au-dessus du {tas_per_thresh}e centile et la précipitation au-dessus du {pr_per_thresh}e centile.", - "title": "Nombre de jours chauds et pluvieux", + "title": "Nombre de jours chauds avec précipitations", "abstract": "Nombre de jours où la température et la précipitation sont au-dessus de centiles donnés." }, "COLD_AND_WET_DAYS": { "long_name": "Nombre de jours où la température est en dessous du {tas_per_thresh}e centile et la précipitation au-dessus du {pr_per_thresh}e centile", "description": "Nombre {freq:m} de jours où la température est en dessous du {tas_per_thresh}e centile et la précipitation au-dessus du {pr_per_thresh}e centile.", - "title": "Nombre de jours froids et pluvieux", + "title": "Nombre de jours froids avec précipitations", "abstract": "Nombre de jours où la température est en dessous d'un centile donné et la précipitation au-dessus d'un centile donné." }, "CALM_DAYS": { diff --git a/src/xclim/data/variables.yml b/src/xclim/data/variables.yml index a02194514..71a7b7dd0 100644 --- a/src/xclim/data/variables.yml +++ b/src/xclim/data/variables.yml @@ -237,7 +237,7 @@ variables: standard_name: lwe_thickness_of_snow_amount data_flags: - negative_accumulation_values: - streamflow: + q: canonical_units: m3 s-1 cell_methods: "time: mean" description: The amount of water, in all phases, flowing in the river channel and flood plain. diff --git a/src/xclim/sdba/adjustment.py b/src/xclim/sdba/adjustment.py index 5c8506071..da5ab54fe 100644 --- a/src/xclim/sdba/adjustment.py +++ b/src/xclim/sdba/adjustment.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +from copy import deepcopy from importlib.util import find_spec from inspect import signature from typing import Any @@ -1818,7 +1819,7 @@ def _adjust( period_dim=None, ): # set default values for non-specified parameters - base_kws_vars = base_kws_vars or {} + base_kws_vars = {} if base_kws_vars is None else deepcopy(base_kws_vars) pts_dim = self.pts_dims[0] for v in sim[pts_dim].values: base_kws_vars.setdefault(v, {}) diff --git a/src/xclim/sdba/base.py b/src/xclim/sdba/base.py index 5036058ef..03ec7d091 100644 --- a/src/xclim/sdba/base.py +++ b/src/xclim/sdba/base.py @@ -5,6 +5,7 @@ from __future__ import annotations +from collections import UserDict from collections.abc import Callable, Sequence from inspect import _empty, signature # noqa @@ -20,7 +21,7 @@ # ## Base class for the sdba module -class Parametrizable(dict): +class Parametrizable(UserDict): """Helper base class resembling a dictionary. This object is _completely_ defined by the content of its internal dictionary, accessible through item access @@ -35,24 +36,23 @@ class Parametrizable(dict): def __getstate__(self): """For (json)pickle, a Parametrizable should be defined by its internal dict only.""" - return self.parameters + return self.data def __setstate__(self, state): """For (json)pickle, a Parametrizable in only defined by its internal dict.""" - self.update(state) + # Unpickling skips the init, so we must _set_ data, we can't just update it - it's not there yet + self.data = {**state} def __getattr__(self, attr): """Get attributes.""" - try: - return self.__getitem__(attr) - except KeyError as err: - # Raise the proper error type for getattr - raise AttributeError(*err.args) from err + if attr == "data" or attr not in self.data: + return self.__getattribute__(attr) + return self.data[attr] @property def parameters(self) -> dict: """All parameters as a dictionary. Read-only.""" - return {**self} + return {**self.data} def __repr__(self) -> str: """Return a string representation.""" diff --git a/src/xclim/testing/utils.py b/src/xclim/testing/utils.py index 338332121..9a1dacec7 100644 --- a/src/xclim/testing/utils.py +++ b/src/xclim/testing/utils.py @@ -13,13 +13,14 @@ import sys import time import warnings -from collections.abc import Sequence +from collections.abc import Callable, Sequence from datetime import datetime as dt +from functools import wraps from importlib import import_module from io import StringIO from pathlib import Path from shutil import copytree -from typing import TextIO +from typing import IO, TextIO from urllib.error import HTTPError, URLError from urllib.parse import urljoin, urlparse from urllib.request import urlretrieve @@ -435,9 +436,28 @@ def load_registry( dict Dictionary of filenames and hashes. """ - remote_registry = audit_url(f"{repo}/{branch}/data/registry.txt") + if not repo.endswith("/"): + repo = f"{repo}/" + remote_registry = audit_url( + urljoin( + urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), + "data/registry.txt", + ) + ) - if branch != default_testdata_version: + if repo != default_testdata_repo_url: + external_repo_name = urlparse(repo).path.split("/")[-2] + external_branch_name = branch.split("/")[-1] + registry_file = Path( + str( + ilr.files("xclim").joinpath( + f"testing/registry.{external_repo_name}.{external_branch_name}.txt" + ) + ) + ) + urlretrieve(remote_registry, registry_file) # noqa: S310 + + elif branch != default_testdata_version: custom_registry_folder = Path( str(ilr.files("xclim").joinpath(f"testing/{branch}")) ) @@ -445,11 +465,9 @@ def load_registry( registry_file = custom_registry_folder.joinpath("registry.txt") urlretrieve(remote_registry, registry_file) # noqa: S310 - elif repo != default_testdata_repo_url: + else: registry_file = Path(str(ilr.files("xclim").joinpath("testing/registry.txt"))) - urlretrieve(remote_registry, registry_file) # noqa: S310 - registry_file = Path(str(ilr.files("xclim").joinpath("testing/registry.txt"))) if not registry_file.exists(): raise FileNotFoundError(f"Registry file not found: {registry_file}") @@ -511,9 +529,13 @@ def nimbus( # noqa: PR01 "The `pooch` package is required to fetch the xclim testing data. " "You can install it with `pip install pooch` or `pip install xclim[dev]`." ) + if not repo.endswith("/"): + repo = f"{repo}/" + remote = audit_url( + urljoin(urljoin(repo, branch if branch.endswith("/") else f"{branch}/"), "data") + ) - remote = audit_url(f"{repo}/{branch}/data") - return pooch.create( + _nimbus = pooch.create( path=cache_dir, base_url=remote, version=default_testdata_version, @@ -522,6 +544,35 @@ def nimbus( # noqa: PR01 registry=load_registry(branch=branch, repo=repo), ) + # Add a custom fetch method to the Pooch instance + # Needed to address: https://github.com/readthedocs/readthedocs.org/issues/11763 + # Fix inspired by @bjlittle (https://github.com/bjlittle/geovista/pull/1202) + _nimbus.fetch_diversion = _nimbus.fetch + + # Overload the fetch method to add user-agent headers + @wraps(_nimbus.fetch_diversion) + def _fetch(*args: str, **kwargs: bool | Callable) -> str: # numpydoc ignore=GL08 + + def _downloader( + url: str, + output_file: str | IO, + poocher: pooch.Pooch, + check_only: bool | None = False, + ) -> None: + """Download the file from the URL and save it to the save_path.""" + headers = {"User-Agent": f"xclim ({__xclim_version__})"} + downloader = pooch.HTTPDownloader(headers=headers) + return downloader(url, output_file, poocher, check_only=check_only) + + # default to our http/s downloader with user-agent headers + kwargs.setdefault("downloader", _downloader) + return _nimbus.fetch_diversion(*args, **kwargs) + + # Replace the fetch method with the custom fetch method + _nimbus.fetch = _fetch + + return _nimbus + # idea copied from raven that it borrowed from xclim that borrowed it from xarray that was borrowed from Seaborn def open_dataset( diff --git a/tests/conftest.py b/tests/conftest.py index ba3597b17..0e3785c93 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ # noqa: D104 from __future__ import annotations +import logging import os from functools import partial from pathlib import Path @@ -411,6 +412,12 @@ def remove_data_written_flag(): """Cleanup cache folder once we are finished.""" flag = default_testdata_cache.joinpath(".data_written") if flag.exists(): - flag.unlink() + try: + flag.unlink() + except FileNotFoundError: + logging.info( + "Teardown race condition occurred: .data_written flag already removed. Lucky!" + ) + pass request.addfinalizer(remove_data_written_flag)