Merge synoptic and autoprocess into the core

.github/workflows/run-tests.yml

@@ -47,7 +47,7 @@ jobs:
         sudo -u postgres psql -U postgres -c "create database openmeteo owner runner"
         python -m pip install --upgrade pip 'setuptools<58'  # See https://stackoverflow.com/questions/69123406
         CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal pip install 'gdal==3.4.1'
-        pip install 'psycopg2>=2.9,<3' codecov coverage isort flake8 'black<25'
+        pip install 'psycopg2>=2.9,<3' codecov coverage isort flake8 'black<25' 'numpy<2'
         pip install -r requirements.txt
         pip install -r requirements-dev.txt
         npm install
@@ -60,7 +60,7 @@ jobs:
         flake8 --max-line-length=88 .
         isort --check-only --diff --profile=black .
         npm run lint
-        coverage run --include="./*" --omit="*/tests/*","*/tests.py","*/migrations/*","./enhydris_project/*" manage.py test -v2
+        coverage run --include="./*" --omit="*/tests/*","*/tests.py","*/migrations/*","./enhydris_project/*","*.pyx" manage.py test -v2
         coverage json
         npm run test
 

.gitignore

@@ -15,7 +15,3 @@ enhydris_project/settings/urls.py
 # enhydris add-ons which may be linked or checked out in the current directory
 enhydris-openhigis
 enhydris_openhigis
-enhydris-autoprocess
-enhydris_autoprocess
-enhydris-synoptic
-enhydris_synoptic

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
-known_third_party=pandas, model_mommy, iso8601, htimeseries, simpletail, rest_captcha, dj_rest_auth, allauth, celery, parler, parler_rest, geowidgets, freezegun
+known_third_party=pandas, model_bakery, iso8601, htimeseries, simpletail, rest_captcha, dj_rest_auth, allauth, celery, parler, parler_rest, geowidgets, freezegun
 known_first_party=enhydris
 known_django=django,rest_framework
 sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER

codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true

doc/dev/database.rst

@@ -488,6 +488,90 @@ Time series and related models
       Returns the number of records inserted (which should be the same
       as the number of records in of ``htimeseries``).
 
+Autoprocess
+-----------
+
+``enhydris.autoprocess`` is an app that automatically processes time
+series to produce new time series. For example, it performs range
+checking, saving a new time series that is range checked.  The app is
+installed by default. If you don't need it, remove it from
+``INSTALLED_APPS``.  When it is installed, in the station page in the
+admin, under "Timeseries Groups", there are some additional options,
+like Range Check, Time Consistency Check, Curve Interpolations and
+Aggregations.
+
+You have a meteorological station called "Hobbiton". It measures
+temperature. Because of sensor, transmission, or other errors,
+sometimes the temperature is wrong—for example, 280 °C. What you want
+to do (and what this app does, among other things) is delete these
+measurements automatically as they come in. In this case, assuming
+that the low and high all-time temperature records in Hobbiton are -18
+and +38 °C, you might decide that anything below -25 or above +50 °C
+(the "hard" limits) is an error, whereas anything below -20 or above
++40 °C (the "soft" limits) is a suspect value. In that case, you
+configure ``enhydris.autoprocess`` with the soft and hard limits. Each
+time data is uploaded, an event is triggered, resulting in an
+asynchronous process processing the initial uploaded data, deleting the
+values outside the hard limits, flagging as suspect the values outside
+the soft limits, and saving the result to the "checked" time series of
+the time series group.
+
+(More specifically, ``enhydris.autoprocess`` uses the ``post_save``
+Django signal for :class:`enhydris.Timeseries` to trigger a Celery task
+that does the auto processing—see ``apps.py`` and ``tasks.py``.)
+
+Range checking is only one of the ways in which a time series can be
+auto-processed—there's also aggregation (e.g. deriving hourly from
+ten-minute time series) and curve interpolation (e.g. deriving discharge
+from stage, or estimating the air speed at a height of 2 m above ground
+when the wind sensor is at a different height). The name we use for all
+these together (i.e. checking, aggregation, interpolation) is "auto
+process". Technically, :class:`AutoProcess` is the super class and it
+has some subclasses such as :class:`Checks`, :class:`Aggregation` and
+:class:`CurveInterpolation`. These are implemented using Django's
+multi-table inheritance. (The checking subclass is called
+:class:`Checks` because there can be many checks—range checking, time
+consistency checking, etc; these are performed one after the other and
+they result in the "checked" time series.)
+
+.. class:: AutoProcess
+
+   .. attribute:: timeseries_group
+
+      The time series group to which this auto-process applies.
+
+   .. method:: execute()
+
+      Performs the auto-processing. It retrieves the new part of the
+      source time series (i.e. the part that starts after the last date
+      of the target time series) and calls the
+      :meth:`process_timeseries` method.
+
+   .. attribute:: source_timeseries
+
+      This is a property; the source time series of the time series
+      group for this auto-process. It depends on the kind of
+      auto-process: for :class:`Checks` it is the initial time series;
+      for :class:`Aggregation` and :class:`CurveInterpolation` it is the
+      checked time series if it exists, or the initial otherwise.  If no
+      suitable time series exists, it is created.
+
+   .. attribute:: target_timeseries
+
+      This is a property; the target time series of the time series
+      group for this auto-process. It depends on the kind of
+      auto-process: for :class:`Checks` it is the checked time series;
+      for :class:`Aggregation` it is the aggregated time series with the
+      target time step; for :class:`CurveInterpolation` it is the
+      initial time series of the target time series group
+      (:class:`CurveInterpolation` has an additional
+      :attr:`target_timeseries_group` attribute). The target time series
+      is created if it does not exist.
+
+   .. method:: process_timeseries()
+
+      Performs the actual processing.
+
 .. _htimeseries: https://github.com/openmeteo/htimeseries
 .. _text format: https://github.com/openmeteo/htimeseries#text-format
 .. _file format: https://github.com/openmeteo/htimeseries#file-format

doc/general/copyright.rst

@@ -5,8 +5,10 @@ Copyright and credits
 
 Enhydris is
 
+| Copyright (C) 2022-2024 IRMASYS P.C.
+| Copyright (C) 2018-2021 Institute of Communication and Computer Systems
 | Copyright (C) 2005-2021 National Technical University of Athens
-| Copyright (C) 2013-2014 TEI of Epirus
+| Copyright (C) 2013-2017 TEI of Epirus
 | Copyright (C) 2015-2017 Antonis Christofides
 | Copyright (C) 2015 Stavros Anastasiadis
 
@@ -29,18 +31,18 @@ Enhydris was funded by several organizations:
   on Enhydris as part of his work at Itia_).
 * In 2009-2010 by the `Ministry of Environment`_ of Greece as part of the
   Hydroscope_ project.
-* In 2013-2014 by the `TEI of Epirus`_ as part of the IRMA_ project.
+* In 2013-2017 by the TEI of Epirus, which later became the `University of
+  Ioannina`_, as part of the IRMA project.
 * In 2015 by GRNET_ as an open technology project.
 * In 2018-2021 by NTUA_ and ICCS_ as part of the OpenHi_ project,
   funded by the EU-Greece_ Sectoral Structural framework "Antagonistikotita_".
 
-.. _ntua: http://www.ntua.gr/
-.. _itia: http://www.itia.ntua.gr/
-.. _ministry of environment: http://ypeka.gr/
+.. _ntua: https://www.ntua.gr/
+.. _itia: https://www.itia.ntua.gr/
+.. _ministry of environment: https://ypeka.gr/
 .. _hydroscope: http://hydroscope.gr/
-.. _tei of epirus: http://www.teiep.gr/en/
-.. _irma: http://www.irrigation-management.eu/
-.. _grnet: https://edet.gr/en
+.. _university of ioannina: https://uoi.gr/
+.. _grnet: https://grnet.gr
 .. _iccs: https://www.iccs.gr
 .. _openhi: https://openhi.net
 .. _eu-greece: https://www.espa.gr

doc/general/install.rst

@@ -355,6 +355,57 @@ Map settings
    lat is in decimal degrees, positive for north/east, negative for
    west/south.
 
+Synoptic settings
+-----------------
+
+``enhydris.synoptic`` is an included app that adds a page to Enhydris
+that shows current conditions in several stations.  It works by creating
+static files which are then served by the web server.
+:data:`ENHYDRIS_SYNOPTIC_ROOT` indicates where these files shall be
+stored. :data:`ENHYDRIS_SYNOPTIC_URL` is currently not used anywhere,
+but it's better to set it anyway; later versions might start to use it
+without warning.
+
+To use ``enhydris.synoptic``, configure your web server to serve
+:data:`ENHYDRIS_SYNOPTIC_ROOT` at :data:`ENHYDRIS_SYNOPTIC_URL`, then go
+to the admin and setup a view.  After celery generates the report, it
+will be available at ``ENHYDRIS_SYNOPTIC_URL + slug + '/'``, where
+``slug`` is the URL identifier given to the synoptic view.
+
+If you do not want to use it ``enhydris.synoptic``, remove it from
+``INSTALLED_APPS``, and remove the ``do-synoptic`` item from
+``CELERY_BEAT_SCHEDULE``.
+
+Note that it does not check permissions; any synoptic view created will
+be public, regardless whether the timeseries from which it is derived
+are marked top secret.
+
+.. data:: ENHYDRIS_SYNOPTIC_ROOT
+
+   The filesystem path where the generated files will be stored (see
+   above).
+
+.. data:: ENHYDRIS_SYNOPTIC_URL
+
+   The URL where the generated files will be served (see above).
+
+.. data:: ENHYDRIS_SYNOPTIC_STATION_LINK_TARGET
+
+   In the rectangles shown on the map, the station name is a link. This
+   is the link target. The default is ``station/{station.id}/`` (the
+   code will use ``.format()`` to replace ``{station.id}`` with the
+   station id).  This default link target leads to a page created by
+   enhydris-synoptic that has a short report about the station, and
+   charts for the last 24 hours. However, in some installations this is
+   undesirable, and it is preferred for the link to lead to the Enhydris
+   station page—in that case, set
+   :data:`ENHYDRIS_SYNOPTIC_STATION_LINK_TARGET` to
+   ``/stations/{station.id}/`` (if the synoptic domain name is different
+   from the main Enhydris domain name, you need to specify the full
+   URL).  (It would be better to use ``django.urls.reverse()`` here
+   instead of a hardwired URL, but it isn't easy to find a general
+   enough solution for all that.)
+
 Miscellaneous settings
 ----------------------
 

doc/general/release-notes.rst

@@ -6,6 +6,39 @@ Release notes
 
 .. highlight:: bash
 
+Development
+===========
+
+Changes from 4.0
+----------------
+
+* Apps ``enhydris-synoptic`` and ``enhydris-autoprocess``, which were
+  third-party, have been included in Enhydris as ``enhydris.synoptic``
+  and ``enhydris.autoprocess``.
+* Instead of having a dependency on ``htimeseries`` (and possibly other
+  modules such as ``haggregate``), we now depend on ``pthelma``, which
+  has all these modules packaged together.
+* Pthelma uses the new pandas time step ("frequency") specifiers, such
+  as ``h`` instead of ``H``.
+
+Upgrading from 4.0
+------------------
+
+If you were not using synoptic and autoprocess, you need to make sure
+that they are removed from ``INSTALLED_APPS`` (in the new version they
+are included by default).
+
+Ensure that you do not have the old ``enhydris-synoptic`` and
+``enhydris-autoprocess`` installed. Also ensure your virtualenv does not
+contain any packages, such as ``htimeseries``, that conflict with
+``pthelma`` (it's better to remove it entirely and recreate it).
+
+Connect to the database (e.g. with ``./manage.py dbshell``) and run
+these commands::
+
+   UPDATE django_migrations SET app='autoprocess' WHERE app='enhydris_autoprocess';
+   UPDATE django_migrations SET app='synoptic' WHERE app='enhydris_synoptic';
+
 Version 4.0
 ===========
 

enhydris/api/tests/test_serializers.py

@@ -5,7 +5,7 @@
 from django.utils import translation
 from rest_framework.test import APITestCase
 
-from model_mommy import mommy
+from model_bakery import baker
 
 from enhydris import models
 from enhydris.api.serializers import (
@@ -20,11 +20,11 @@
 
 class StationSerializerTestCase(APITestCase):
     def setUp(self):
-        station = mommy.make(
+        station = baker.make(
             models.Station,
             name="hello",
             overseer="Bilbo Baggins",
-            maintainers=[mommy.make(User, username="Bilbo")],
+            maintainers=[baker.make(User, username="Bilbo")],
         )
         self.serializer = StationSerializer(station)
 
@@ -51,7 +51,7 @@ def test_last_update_nonempty(self, m):
 
 class TimeseriesGroupSerializerTestCase(APITestCase):
     def setUp(self):
-        timeseries_group = mommy.make(
+        timeseries_group = baker.make(
             models.TimeseriesGroup,
             name="My time series group",
         )
@@ -64,7 +64,7 @@ def test_name(self):
 class TimeseriesSerializerTestCase(APITestCase):
     @classmethod
     def setUpTestData(cls):
-        cls.timeseries_group = mommy.make(models.TimeseriesGroup)
+        cls.timeseries_group = baker.make(models.TimeseriesGroup)
 
     def test_type_serialization_for_initial(self):
         serializer = TimeseriesSerializer(models.Timeseries(type=100))
@@ -105,7 +105,7 @@ def test_type_deserialization_for_checked(self):
 class TimeseriesSerializerUniqueTypeTestCase(APITestCase):
     @classmethod
     def setUpTestData(cls):
-        cls.timeseries_group = mommy.make(
+        cls.timeseries_group = baker.make(
             models.TimeseriesGroup,
             variable__descr="Temperature",
         )
@@ -124,7 +124,7 @@ def test_only_one_regularized_timeseries_per_group(self):
 
     def _create_timeseries(self, type):
         """Create a time series of the specified type with empty time step."""
-        self.timeseries = mommy.make(
+        self.timeseries = baker.make(
             models.Timeseries,
             timeseries_group=self.timeseries_group,
             type=type,

enhydris/api/tests/test_views/test_bounding_box.py

@@ -2,20 +2,20 @@
 from django.test import override_settings
 from rest_framework.test import APITestCase
 
-from model_mommy import mommy
+from model_bakery import baker
 
 from enhydris import models
 
 
 @override_settings(ENHYDRIS_AUTHENTICATION_REQUIRED=False)
 class SimpleBoundingBoxTestCase(APITestCase):
     def setUp(self):
-        mommy.make(
+        baker.make(
             models.Station,
             geom=Point(x=21.06071, y=39.09518, srid=4326),
             original_srid=4326,
         )
-        mommy.make(
+        baker.make(
             models.Station,
             geom=Point(x=21.60121, y=39.22440, srid=4326),
             original_srid=4326,
@@ -60,12 +60,12 @@ def test_y2(self):
 @override_settings(ENHYDRIS_AUTHENTICATION_REQUIRED=False)
 class TooSmallBoundingBoxTestCase(APITestCase):
     def setUp(self):
-        mommy.make(
+        baker.make(
             models.Station,
             geom=Point(x=21.06071, y=39.09518, srid=4326),
             original_srid=4326,
         )
-        mommy.make(
+        baker.make(
             models.Station,
             geom=Point(x=21.60121, y=39.22440, srid=4326),
             original_srid=4326,