diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41d02e4..b3e65c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,13 +13,13 @@ jobs: - uses: actions/checkout@main - uses: actions/setup-python@main with: - python-version: '3.x' + python-version: "3.x" - uses: actions/cache@main with: path: ${{ env.pythonLocation }} key: build-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml', 'setup.*') }} - run: pip wheel . --no-deps -w dist - - uses: pypa/gh-action-pypi-publish@master + - uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/install_from_pypi.yml b/.github/workflows/install_from_pypi.yml new file mode 100644 index 0000000..047b949 --- /dev/null +++ b/.github/workflows/install_from_pypi.yml @@ -0,0 +1,38 @@ +name: "Test installation from PyPI" + +on: + workflow_dispatch: + schedule: + - cron: "4 5 * * *" # Every day at 05:04 + +jobs: + test_pypi_installation: + name: test PyPI installation + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: "ubuntu-latest" + python: "3.12" + steps: + - uses: actions/checkout@main + - uses: actions/setup-python@main + with: + python-version: ${{ matrix.python }} + # Debug + - run: type -a python + - run: python --version + - run: python -m pip --version + - run: python -m pip cache info + # Install the package from pypi + - run: python -m pip install searvey + - run: python -m pip freeze + # Checkout the version of code that got installed from PyPI + - run: git fetch --tags + - run: git checkout v$(python -c 'import importlib.metadata; print(importlib.metadata.version("searvey"))') + # Install test dependencies + - run: pip install -U $(cat requirements/requirements-dev.txt| grep --extended-regexp 'pytest=|pytest-recording=|urllib3=' | cut -d ';' -f1) + # Remove the source code (just to be sure that it is not being used) + - run: rm -rf searvey + # Run the tests + - run: make test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdaf062..053a4d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,7 +41,7 @@ repos: - id: "shellcheck" - repo: "https://github.com/python-jsonschema/check-jsonschema" - rev: "0.28.5" + rev: "0.28.6" hooks: - id: "check-github-workflows" - id: "check-readthedocs" @@ -60,7 +60,7 @@ repos: - repo: "https://github.com/charliermarsh/ruff-pre-commit" # Ruff version. - rev: 'v0.4.9' + rev: 'v0.4.10' hooks: - id: "ruff" diff --git a/Makefile b/Makefile index 0e3115e..3edd915 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ clean_notebooks: pre-commit run nbstripout -a exec_notebooks: - pytest --nbmake --nbmake-timeout=60 --nbmake-kernel=python3 $$(git ls-files | grep ipynb) + pytest --ff --nbmake --nbmake-timeout=90 --nbmake-kernel=python3 $$(git ls-files | grep ipynb) docs: make -C docs html diff --git a/examples/IOC_data.ipynb b/examples/IOC_data.ipynb index 89844b9..e8b0a8c 100644 --- a/examples/IOC_data.ipynb +++ b/examples/IOC_data.ipynb @@ -30,7 +30,7 @@ "source": [ "## Retrieve Station Metadata\n", "\n", - "In order to retrieve station metadata we need to use the `get_ioc_stations()` function:" + "In order to retrieve station metadata we need to use the `get_ioc_stations()` function which returns a `geopandas.GeoDataFrame`:" ] }, { @@ -55,7 +55,7 @@ }, "outputs": [], "source": [ - "ioc_stations.sample(3).sort_index()" + "ioc_stations.columns" ] }, { @@ -67,7 +67,8 @@ }, "outputs": [], "source": [ - "ioc_stations.columns" + "with pd.option_context('display.max_columns', None):\n", + " ioc_stations.sample(3).sort_index()" ] }, { @@ -105,8 +106,8 @@ "outputs": [], "source": [ "east_coast = shapely.geometry.box(-85, 25, -65, 45)\n", - "east_stations = searvey.get_ioc_stations(region=east_coast)\n", - "len(east_stations)" + "east_coast_stations = searvey.get_ioc_stations(region=east_coast)\n", + "len(east_coast_stations)" ] }, { @@ -118,7 +119,7 @@ }, "outputs": [], "source": [ - "east_stations.hvplot.points(geo=True, tiles=True)" + "east_coast_stations.hvplot.points(geo=True, tiles=True)" ] }, { @@ -128,7 +129,7 @@ "source": [ "## Retrieve IOC station data\n", "\n", - "The function for retrieving data is called `fetch_ioc_station()`. \n", + "The function for retrieving data is called `fetch_ioc_station()` and it returns \n", "\n", "In its simplest form it only requires the station_id (i.e. IOC_CODE) and it will retrieve the last week of data:" ] @@ -163,7 +164,6 @@ " station_id=\"alva\",\n", " start_date=pd.Timestamp(\"2024-05-01\"),\n", " end_date=pd.Timestamp(\"2024-05-10\"),\n", - " progress_bar=False,\n", ")\n", "df" ] @@ -175,7 +175,7 @@ "source": [ "If we request more than 30 days, then multiple HTTP requests are send to the IOC servers via multithreading and the responses are merged to a single dataframe. \n", "\n", - "In this case, setting `progress_bar=True` can be useful in monitoring the progress of HTTP requests. \n", + "In this case, setting `progress_bar=True` can be helpful in monitoring the progress of the HTTP requests. \n", "For example to retrieve data for the first 6 months of 2020:" ] }, @@ -200,7 +200,7 @@ "id": "15", "metadata": {}, "source": [ - "Keep in mind that each IOC station may return dataframes with different sensors/columns. For example the station in Bahamas returns a bunch of them:" + "Keep in mind that each IOC station may return dataframes with different sensors/columns. For example the `setp1` station in Bahamas returns a bunch of them:" ] }, { @@ -224,7 +224,7 @@ "id": "17", "metadata": {}, "source": [ - "Furthermore, not all of these timeseries are ready to be used. \n", + "Nevertheless, the returned timeseries are **not** ready to be used. \n", "\n", "E.g. we see that in the last days of May the `rad` sensor was offline for some time:" ] @@ -236,7 +236,15 @@ "metadata": {}, "outputs": [], "source": [ - "bahamas.rad.hvplot()" + "bahamas.rad.hvplot(grid=True)" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "So the IOC data **do** need some data-cleaning." ] } ], diff --git a/poetry.lock b/poetry.lock index c5b4d58..9165252 100644 --- a/poetry.lock +++ b/poetry.lock @@ -665,13 +665,13 @@ tests = ["pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "dask" -version = "2024.6.0" +version = "2024.6.2" description = "Parallel PyData with Task Scheduling" optional = false python-versions = ">=3.9" files = [ - {file = "dask-2024.6.0-py3-none-any.whl", hash = "sha256:de0ced6cd46dbc6c01120c8870457af46d667940805a4be063a74dd467466804"}, - {file = "dask-2024.6.0.tar.gz", hash = "sha256:6882ce7e485336d707e540080ed48e01f9c09485d52a2928ea05f9a9e44bb433"}, + {file = "dask-2024.6.2-py3-none-any.whl", hash = "sha256:81b80ee015b2e057b93bb2d1bf13a866136e762e2b24bf54b6b621e8b86b7708"}, + {file = "dask-2024.6.2.tar.gz", hash = "sha256:d429d6b19e85fd1306ac37c188aaf99d03bbe69a6fe59d2b42882b2ac188686f"}, ] [package.dependencies] @@ -689,7 +689,7 @@ array = ["numpy (>=1.21)"] complete = ["dask[array,dataframe,diagnostics,distributed]", "lz4 (>=4.3.2)", "pyarrow (>=7.0)", "pyarrow-hotfix"] dataframe = ["dask-expr (>=1.1,<1.2)", "dask[array]", "pandas (>=1.3)"] diagnostics = ["bokeh (>=2.4.2)", "jinja2 (>=2.10.3)"] -distributed = ["distributed (==2024.6.0)"] +distributed = ["distributed (==2024.6.2)"] test = ["pandas[test]", "pre-commit", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist"] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 338c254..a09b32a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ sphinx = "*" sphinx-autodoc-typehints = "*" sphinxext-opengraph = "*" toml = "*" +sphinx-autodoc-typehints = "*" [tool.poetry.group.jupyter.dependencies] hvplot = {version = "*", extras = ["geo"]} diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 224e66e..32a20d8 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -25,7 +25,7 @@ covdefaults==2.3.0 ; python_version >= "3.9" and python_version < "4.0" coverage==7.5.3 ; python_version >= "3.9" and python_version < "4.0" coverage[toml]==7.5.3 ; python_version >= "3.9" and python_version < "4.0" cycler==0.12.1 ; python_version >= "3.9" and python_version < "4.0" -dask==2024.6.0 ; python_version >= "3.9" and python_version < "4.0" +dask==2024.6.2 ; python_version >= "3.9" and python_version < "4.0" dataretrieval==1.0.9 ; python_version >= "3.9" and python_version < "4.0" debugpy==1.8.1 ; python_version >= "3.9" and python_version < "4.0" decorator==5.1.1 ; python_version >= "3.9" and python_version < "4.0"