Merge pull request #168 from pacificclimate/i167-upgrade-python

I167 upgrade python

.bumpversion.cfg

@@ -0,0 +1,20 @@
+[bumpversion]
+current_version = 1.2.3
+commit = True
+tag = True
+
+[bumpversion:file:thunderbird/__version__.py]
+search = __version__ = "{current_version}"
+replace = __version__ = "{new_version}"
+
+[bumpversion:file:pyproject.toml]
+search = version = "{current_version}"
+replace = version = "{new_version}"
+
+[bumpversion:file:Dockerfile]
+search = Version="{current_version}"
+replace = Version="{new_version}"
+
+[bumpversion:file:.cruft.json]
+search = "version": "{current_version}"
+replace = "version": "{new_version}"

.cruft.json

@@ -11,7 +11,7 @@
             "project_repo_name": "thunderbird",
             "project_readthedocs_name": "thunderbird",
             "project_short_description": "A Web Processing Service for Climate Explorer data preparation",
-            "version": "0.6.0",
+            "version": "1.2.3",
             "open_source_license": "GNU General Public License v3",
             "http_port": "5000",
             "_copy_without_render": [
@@ -20,4 +20,4 @@
             "_template": "https://github.com/bird-house/cookiecutter-birdhouse"
         }
     }
-}
+}

.env

@@ -0,0 +1 @@
+DACCS_HOST="marble-dev01.pcic.uvic.ca"

.github/workflows/python-ci.yml

@@ -8,24 +8,30 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
     - name: Install dependencies
       run: |
+        poetry env use python${{ matrix.python-version }}
         make install-ci
     - name: Test with pytest (full)
       if: github.ref == 'refs/heads/master'
       run: |
-        py.test -m "not online" -v
+        make test-not-online
     - name: Test with pytest (fast)
       if: github.ref != 'refs/heads/master'
       run: |
-        py.test -m "not online and not slow" -v
+        make test
     - name: Code Quality
-      run: black . --check
+      run: |
+        poetry run black . --check

.gitignore

@@ -78,7 +78,6 @@ src/
 
 # tests
 *.log
-*.lock
 testdata.json
 
 # IPython

.ncrc

@@ -0,0 +1 @@
+HTTP.SSL.CAINFO=/etc/ssl/certs/ca-certificates.crt

Dockerfile

@@ -1,25 +1,19 @@
-FROM python:3.8-slim
+FROM python:3.10-slim
 
 LABEL Maintainer=https://github.com/pacificclimate/thunderbird Description="thunderbird WPS" Vendor="pcic" Version="1.2.3"
 ENV DEBIAN_FRONTEND=noninteractive
 ENV PIP_INDEX_URL="https://pypi.pacificclimate.org/simple/"
 
+RUN apt-get update && \
+    apt-get install -y cdo curl libhdf5-serial-dev netcdf-bin libnetcdf-dev && \
+    rm -rf /var/lib/apt/lists/*
+
 COPY . /tmp
 WORKDIR /tmp
 
-RUN apt-get update && \
-    apt-get install -y build-essential cdo libhdf5-serial-dev netcdf-bin libnetcdf-dev && \
-    rm -rf /var/lib/apt/lists/* && \
-    # this line combats the issue found here:
-    # https://superuser.com/questions/1347723/arch-on-wsl-libqt5core-so-5-not-found-despite-being-installed
-    strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5 && \
-    # Removing this package prevents a scan failure. It will be reintroduced
-    # once 1.24.2 becomes available through the base image.
-    # Issue link: https://github.com/advisories/GHSA-mh33-7rrq-662w
-    apt-get remove -y python3-urllib3 && \
-    pip install -U pip && \
-    pip install . && \
-    pip install gunicorn
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH=/root/.local/bin:$PATH
+RUN poetry install
 
 EXPOSE 5000
-CMD ["gunicorn", "--bind=0.0.0.0:5000", "thunderbird.wsgi:application"]
+CMD ["poetry", "run", "gunicorn", "-t 60", "--bind=0.0.0.0:5000", "thunderbird.wsgi:application"]

Makefile

@@ -1,19 +1,16 @@
 # Configuration
 APP_ROOT := $(abspath $(lastword $(MAKEFILE_LIST))/..)
 APP_NAME := thunderbird
-VENV?=/tmp/thunderbird-venv
-PYTHON=${VENV}/bin/python3
-PIP=${VENV}/bin/pip
-export PIP_INDEX_URL=https://pypi.pacificclimate.org/simple
+include .env
 
 # Notebook targets
-LOCAL_URL = http://localhost:5000
+LOCAL_URL = http://localhost:5000/wps
 DEV_PORT ?= $(shell bash -c 'read -ep "Target port: " port; echo $$port')
 
 # Used in target refresh-notebooks to make it looks like the notebooks have
 # been refreshed from the production server below instead of from the local dev
 # instance so the notebooks can also be used as tutorial notebooks.
-OUTPUT_URL = https://docker-dev03.pcic.uvic.ca/wpsoutputs
+OUTPUT_URL = https://$(DACCS_HOST)/wpsoutputs
 SANITIZE_FILE := https://github.com/Ouranosinc/PAVICS-e2e-workflow-tests/raw/master/notebooks/docs/output-sanitize.cfg
 
 
@@ -32,8 +29,9 @@ help:
 	@echo "  status            to show status of $(APP_NAME) service."
 	@echo "  clean             to remove all files generated by build and tests."
 	@echo "\nTesting targets:"
-	@echo "  test              to run tests (but skip long running tests)."
-	@echo "  test-all          to run all tests (including long running tests)."
+	@echo "  test              to run tests (but skip online and long running tests)."
+	@echo "  test-not-online   to run tests (but skip online tests)."
+	@echo "  test-all          to run all tests (including online and long running tests)."
 	@echo "  lint              to run code style checks with flake8."
 	@echo "\nSphinx targets:"
 	@echo "  docs              to generate HTML documentation with Sphinx."
@@ -43,9 +41,9 @@ help:
 ## Build targets
 
 .PHONY: install
-install: install-cdo venv
+install: install-cdo
 	@echo "Installing application ..."
-	@-bash -c '${PIP} install -e .'
+	@-bash -c 'poetry install'
 	@echo "\nStart service with \`thunderbird start'"
 
 .PHONY: install-cdo
@@ -57,13 +55,12 @@ install-cdo:
 .PHONY: install-ci
 install-ci: install-cdo
 	@echo "Installing ci requirements"
-	@-bash -c 'pip install -r requirements.txt'
-	@-bash -c 'pip install -e ".[dev]"'
+	@-bash -c 'poetry install --with=dev'
 
 .PHONY: develop
-develop: install-cdo venv
+develop: install-cdo
 	@echo "Installing development requirements for tests and docs ..."
-	@-bash -c '${PIP} install -e ".[dev]"'
+	@-bash -c 'poetry install --with=dev'
 
 # These commands have been disabled until further notice. Due to the issues with
 # cdo's signal handling, we cannot reliably start/stop using make.
@@ -121,21 +118,22 @@ clean-dist: clean
 	## do not use git clean -e/--exclude here, add them to .gitignore instead
 	@-git clean -dfx
 
-.PHONY: venv
-venv:
-	test -d $(VENV) || python3 -m venv $(VENV)
-
 ## Test targets
 
 .PHONY: test
-test: venv
+test:
 	@echo "Running tests (skip slow and online tests) ..."
-	@bash -c '${PYTHON} -m pytest -v -m "not slow and not online" tests/'
+	@bash -c 'poetry run pytest -v -m "not slow and not online" tests/'
 
 .PHONY: test-all
-test-all: venv
+test-all:
 	@echo "Running all tests (including slow and online tests) ..."
-	@bash -c '${PYTHON} -m pytest -v tests/'
+	@bash -c 'poetry run pytest -v tests/'
+
+.PHONY: test-not-online
+test-not-online:
+	@echo "Running tests (including slow but excluding online tests) ..."
+	@bash -c 'poetry run pytest -v -m "not online" tests/'
 
 .PHONY: notebook-sanitizer
 notebook-sanitizer:
@@ -145,40 +143,39 @@ notebook-sanitizer:
 .PHONY: test-notebooks
 test-notebooks: notebook-sanitizer
 	@echo "Running notebook-based tests"
-	@bash -c "source $(VENV)/bin/activate && env LOCAL_URL=$(LOCAL_URL) pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
+	@bash -c "poetry shell && env LOCAL_URL=$(LOCAL_URL) pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
 
 .PHONY: test-notebooks-prod
 test-notebooks-prod: notebook-sanitizer
 	@echo "Running notebook-based tests against production instance of thunderbird"
-	@bash -c "source $(VENV)/bin/activate && pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
+	@bash -c "poetry shell && pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
 
 .PHONY: test-notebooks-dev
 test-notebooks-dev: notebook-sanitizer
 	@echo "Running notebook-based tests against development instance of thunderbird"
-	@bash -c "source $(VENV)/bin/activate && env DEV_URL=http://docker-dev03.pcic.uvic.ca:30099/wps pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
+	@bash -c "poetry shell && env DEV_URL=http://$(DACCS_HOST):30099/wps pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
 
 .PHONY: test-notebooks-custom
 test-notebooks-custom: notebook-sanitizer
 	@echo "Running notebook-based tests against custom instance of thunderbird"
-	@bash -c "source $(VENV)/bin/activate && env DEV_URL=http://docker-dev03.pcic.uvic.ca:$(DEV_PORT)/wps pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
+	@bash -c "poetry shell && env DEV_URL=http://$(DACCS_HOST):$(DEV_PORT)/wps pytest --nbval --verbose $(CURDIR)/notebooks/ --sanitize-with $(CURDIR)/docs/output-sanitize.cfg --ignore $(CURDIR)/notebooks/.ipynb_checkpoints"
 
 .PHONY: lint
 lint: venv
 	@echo "Running black code style checks ..."
-	@bash -c '${PYTHON} -m black . --check'
+	@bash -c 'poetry run black . --check'
 
 ## Sphinx targets
 
 .PHONY: docs
 docs:
 	@echo "Updating notebook docs"
-	@bash -c 'source $(VENV)/bin/activate && jupyter nbconvert --to html notebooks/* --output-dir docs/formatted_demos/'
+	@bash -c 'poetry shell && jupyter nbconvert --to html notebooks/* --output-dir docs/formatted_demos/'
 
 ## Deployment targets
 
 .PHONY: dist
 dist: clean
 	@echo "Builds source and wheel package ..."
-	@-python3 setup.py sdist
-	@-python3 setup.py bdist_wheel
+	@bash -c 'poetry build'
 	ls -l dist

docs/configuration.md

@@ -6,13 +6,13 @@
 You can overwrite the default [PyWPS](http://pywps.org/) configuration by using command-line options.
 See the thunderbird help which options are available:
 ```
-(venv)$ thunderbird start --help
+(thunderbird-py<python_version>)$ thunderbird start --help
 --hostname HOSTNAME        hostname in PyWPS configuration.
 --port PORT                port in PyWPS configuration.
 ```
 Start service with different hostname and port:
 ```
-(venv)$ thunderbird start --hostname localhost --port 5000
+(thunderbird-py<python_version>)$ thunderbird start --hostname localhost --port 5000
 ```
 
 ## Use a custom configuration file
@@ -33,5 +33,5 @@ level = DEBUG
 Start the service with your custom configuration:
 ```
 # start the service with this configuration
-(venv)$ thunderbird start -c etc/custom.cfg
+(thunderbird-py<python_version>)$ thunderbird start -c etc/custom.cfg
 ```

docs/dev_guide.md

@@ -14,32 +14,32 @@ $ make docs
 ## Running tests
 Run tests using [`pytest`](https://docs.pytest.org/en/latest/).
 
-First activate the `thunderbird` Python environment and install `pytest`.
+First install the `thunderbird` Python environment.
 ```
-$ python3 -m venv venv
-$ source venv/bin/activate
-(venv)$ pip install -r requirements_dev.txt  # if not already installed
+$ poetry install --with=dev
+$ poetry shell
 # OR
-(venv)$ make install
+$ make develop
+$ poetry shell
 ```
 
 Run quick tests (skip slow and online):
 ```
-(venv)$ pytest -m 'not slow and not online'"
+(thunderbird-py<python_version>)$ pytest -m 'not slow and not online'"
 ```
 Run all tests:
 ```
-(venv)$ pytest
+(thunderbird-py<python_version>)$ pytest
 ```
 
 You can also run tests on the notebooks using the `Makefile`.
 ```
-$ make test-notebooks
+$ make test-notebooks-prod
 ```
 
 Check `black` formatting:
 ```
-(venv)$ black .
+(thunderbird-py<python_version>)$ black .
 ```
 
 ## Run tests the lazy way
@@ -54,11 +54,10 @@ $ make lint
 Make a new version of `thunderbird` in the following steps:
 
 * Make sure everything is committed to GitHub.
-* Update `CHANGES.md` with the next version.
-* Dry Run: `bumpversion --dry-run --verbose --new-version 0.8.1 patch`
-* Do it: `bumpversion --new-version 0.8.1 patch`
-* ... or: `bumpversion --new-version 0.9.0 minor`
-* Push it: `git push`
-* Push tag: `git push --tags`
+* Update `CHANGES.md` with the next version and commit.
+* Dry Run: `bumpversion --dry-run --verbose patch`
+* Do it: `bumpversion patch`
+* ... or: `bumpversion minor`
+* Push it: `git push --tags`
 
 See the [bumpversion](https://pypi.org/project/bumpversion/) documentation for details.