Merge pull request #310 from ropable/master

Add SSE view for device location, switch to use ASGI server, bugfix DeviceResource

Dockerfile

@@ -49,4 +49,4 @@ COPY tracking ./tracking
 RUN python manage.py collectstatic --noinput
 USER ${UID}
 EXPOSE 8080
-CMD ["gunicorn", "resource_tracking.wsgi", "--config", "gunicorn.py"]
+CMD ["gunicorn", "resource_tracking.asgi:application", "--config", "gunicorn.py"]

README.md

@@ -47,9 +47,9 @@ Other environment variables will be required to run the project in production
 
 ## Running
 
-Use `runserver` to run a local copy of the application:
+Use `gunicorn` to run the local ASGI server (`runserver` doesn't support async responses yet):
 
-    python manage.py runserver 0:8080
+    gunicorn resource_tracking.asgi:application --config gunicorn.py --reload
 
 Run console commands manually:
 

SECURITY.md

@@ -7,7 +7,7 @@ source code repositories managed through our GitHub organisation
 
 This repository takes guidance relating to Secure Software Development from the
 [WA Government Cyber Security
-Policy](https://www.wa.gov.au/system/files/2022-01/WA%20Government%20Cyber%20Security%20Policy.pdf).
+Policy](https://www.wa.gov.au/government/publications/2024-wa-government-cyber-security-policy).
 
 If you believe that you have found a security vulnerability in any DBCA-managed
 repository, please report it to us as described below.
@@ -25,13 +25,13 @@ do not, please follow up via email to ensure we received your original message.
 Please include the requested information listed below (as much as you can provide)
 to help us better understand the nature and scope of the possible issue:
 
-  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
-  * Full paths of source file(s) related to the manifestation of the issue
-  * The location of the affected source code (tag/branch/commit or direct URL)
-  * Any special configuration required to reproduce the issue
-  * Step-by-step instructions to reproduce the issue
-  * Proof-of-concept or exploit code (if possible)
-  * Impact of the issue, including how an attacker might exploit the issue
+- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
 
 This information will help us triage your report more quickly. Please note that
 we prefer all communications to be in English.

gunicorn.py

@@ -12,3 +12,5 @@
 timeout = 180
 # Disable access logging.
 accesslog = None
+# Use UvicornWorker as the worker class.
+worker_class = "resource_tracking.workers.UvicornWorker"

kustomize/overlays/prod/kustomization.yaml

@@ -29,4 +29,4 @@ patches:
   - path: service_patch.yaml
 images:
   - name: ghcr.io/dbca-wa/resource_tracking
-    newTag: 1.4.21
+    newTag: 1.4.22

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "resource_tracking"
-version = "1.4.21"
+version = "1.4.22"
 description = "DBCA internal corporate application to download and serve data from remote tracking devices."
 authors = ["DBCA OIM <[email protected]>"]
 license = "Apache-2.0"
@@ -20,10 +20,13 @@ django-geojson = "4.1.0"
 unicodecsv = "0.14.1"
 whitenoise = { version = "6.8.2", extras = ["brotli"] }
 azure-storage-blob = "12.23.1"
-sentry-sdk = { version = "2.19.0", extras = ["django"] }
+sentry-sdk = {version = "2.19.2", extras = ["django"]}
+uvicorn = { extras = ["standard"], version = "^0.34.0" }
+uvicorn-worker = "^0.3.0"
+orjson = "3.10.13"
 
 [tool.poetry.group.dev.dependencies]
-ipython = "^8.30.0"
+ipython = "^8.31.0"
 ipdb = "^0.13.13"
 pre-commit = "^4.0.1"
 mixer = "^7.2.2"

resource_tracking/asgi.py

@@ -0,0 +1,20 @@
+"""
+ASGI config for resource_tracking project.
+It exposes the ASGI callable as a module-level variable named ``application``.
+"""
+
+import os
+from pathlib import Path
+
+from django.core.asgi import get_asgi_application
+
+# These lines are required for interoperability between local and container environments.
+d = Path(__file__).resolve().parent.parent
+dot_env = os.path.join(str(d), ".env")
+if os.path.exists(dot_env):
+    from dotenv import load_dotenv
+
+    load_dotenv()
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "resource_tracking.settings")
+application = get_asgi_application()

resource_tracking/workers.py

@@ -0,0 +1,9 @@
+from typing import Any, Dict
+
+from uvicorn.workers import UvicornWorker as BaseUvicornWorker
+
+
+class UvicornWorker(BaseUvicornWorker):
+    # UvicornWorker doesn't support the lifespan protocol.
+    # Reference: https://stackoverflow.com/a/75996092/14508
+    CONFIG_KWARGS: Dict[str, Any] = {"loop": "auto", "http": "auto", "lifespan": "off"}

resource_tracking/wsgi.py

@@ -2,15 +2,18 @@
 WSGI config for resource_tracking project.
 It exposes the WSGI callable as a module-level variable named ``application``.
 """
+
 import os
-from django.core.wsgi import get_wsgi_application
 from pathlib import Path
 
+from django.core.wsgi import get_wsgi_application
+
 # These lines are required for interoperability between local and container environments.
-d = Path(__file__).resolve().parent
-dot_env = os.path.join(str(d), '.env')
+d = Path(__file__).resolve().parent.parent
+dot_env = os.path.join(str(d), ".env")
 if os.path.exists(dot_env):
     from dotenv import load_dotenv
+
     load_dotenv()
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "resource_tracking.settings")

tracking/api.py

@@ -1,33 +1,32 @@
-from datetime import datetime, timedelta
+from datetime import timedelta
+from io import BytesIO
+
+import unicodecsv as csv
 from django.conf import settings
 from django.core.exceptions import FieldError
 from django.urls import path
 from django.utils import timezone
-from io import BytesIO
 from tastypie import fields
 from tastypie.cache import NoCache
 from tastypie.http import HttpBadRequest
-from tastypie.resources import ModelResource, ALL_WITH_RELATIONS
+from tastypie.resources import ALL_WITH_RELATIONS, ModelResource
 from tastypie.serializers import Serializer
 from tastypie.utils import format_datetime
-import unicodecsv as csv
 
 from tracking.models import Device
 
 
 class CSVSerializer(Serializer):
-    formats = settings.TASTYPIE_DEFAULT_FORMATS + ['csv']
+    formats = settings.TASTYPIE_DEFAULT_FORMATS + ["csv"]
 
-    content_types = dict(
-        Serializer.content_types.items() |
-        [('csv', 'text/csv')])
+    content_types = dict(Serializer.content_types.items() | [("csv", "text/csv")])
 
     def format_datetime(self, data):
         # Override the default `format_datetime` method of the class
         # to return datetime as timezone-aware.
-        if self.datetime_formatting == 'rfc-2822':
+        if self.datetime_formatting == "rfc-2822":
             return format_datetime(data)
-        if self.datetime_formatting == 'iso-8601-strict':
+        if self.datetime_formatting == "iso-8601-strict":
             # Remove microseconds to strictly adhere to iso-8601
             data = data - timedelta(microseconds=data.microsecond)
 
@@ -37,12 +36,12 @@ def to_csv(self, data, options=None):
         options = options or {}
         data = self.to_simple(data, options)
         raw_data = BytesIO()
-        if 'objects' in data and data['objects']:
-            fields = data['objects'][0].keys()
-            writer = csv.DictWriter(raw_data, fields, dialect='excel', extrasaction='ignore')
+        if "objects" in data and data["objects"]:
+            fields = data["objects"][0].keys()
+            writer = csv.DictWriter(raw_data, fields, dialect="excel", extrasaction="ignore")
             header = dict(zip(fields, fields))
             writer.writerow(header)
-            for item in data['objects']:
+            for item in data["objects"]:
                 writer.writerow(item)
 
         return raw_data.getvalue()
@@ -67,29 +66,28 @@ def generate_filtering(mdl):
 
 def generate_meta(klass, overrides={}):
     metaitems = {
-        'queryset': klass.objects.all(),
-        'resource_name': klass._meta.model_name,
-        'filtering': generate_filtering(klass),
+        "queryset": klass.objects.all(),
+        "resource_name": klass._meta.model_name,
+        "filtering": generate_filtering(klass),
     }
     metaitems.update(overrides)
-    return type('Meta', (object,), metaitems)
+    return type("Meta", (object,), metaitems)
 
 
 class APIResource(ModelResource):
-
     def prepend_urls(self):
         return [
             path(
-                "<resource_name>/fields/<field_name>/".format(self._meta.resource_name),
-                self.wrap_view('field_values'), name="api_field_values"),
+                "<resource_name>/fields/<field_name>/".format(), self.wrap_view("field_values"), name="api_field_values"
+            ),
         ]
 
     def field_values(self, request, **kwargs):
         # Get a list of unique values for the field passed in kwargs.
         try:
-            qs = self._meta.queryset.values_list(kwargs['field_name'], flat=True).distinct()
+            qs = self._meta.queryset.values_list(kwargs["field_name"], flat=True).distinct()
         except FieldError as e:
-            return self.create_response(request, data={'error': str(e)}, response_class=HttpBadRequest)
+            return self.create_response(request, data={"error": str(e)}, response_class=HttpBadRequest)
         # Prepare return the HttpResponse.
         return self.create_response(request, data=list(qs))
 
@@ -98,8 +96,8 @@ class HttpCache(NoCache):
     """
     Just set the cache control header to implement web cache
     """
-    def __init__(self, timeout=0, public=None,
-                 private=None, *args, **kwargs):
+
+    def __init__(self, timeout=0, public=None, private=None, *args, **kwargs):
         """
         Optionally accepts a ``timeout`` in seconds for the resource's cache.
         Defaults to ``0`` seconds.
@@ -111,8 +109,8 @@ def __init__(self, timeout=0, public=None,
 
     def cache_control(self):
         control = {
-            'max_age': self.timeout,
-            's_maxage': self.timeout,
+            "max_age": self.timeout,
+            "s_maxage": self.timeout,
         }
 
         if self.public is not None:
@@ -125,26 +123,27 @@ def cache_control(self):
 
 
 class DeviceResource(APIResource):
-
-    def build_filters(self, filters=None):
-        """Override build_filters to allow filtering by seen_age__lte=<minutes>
-        """
+    def build_filters(self, filters=None, **kwargs):
+        """Override build_filters to allow filtering by seen_age__lte=<minutes>"""
         if filters is None:
             filters = {}
         orm_filters = super(DeviceResource, self).build_filters(filters)
 
-        if 'seen_age__lte' in filters:
+        if "seen_age__lte" in filters:
             # Convert seen_age__lte to a timedelta
-            td = timedelta(minutes=int(filters['seen_age__lte']))
-            orm_filters['seen__gte'] = timezone.now() - td
+            td = timedelta(minutes=int(filters["seen_age__lte"]))
+            orm_filters["seen__gte"] = timezone.now() - td
 
         return orm_filters
 
-    Meta = generate_meta(Device, {
-        'cache': HttpCache(settings.DEVICE_HTTP_CACHE_TIMEOUT),
-        'serializer': CSVSerializer(),
-    })
-    age_minutes = fields.IntegerField(attribute='age_minutes', readonly=True, null=True)
-    age_colour = fields.CharField(attribute='age_colour', readonly=True, null=True)
-    age_text = fields.CharField(attribute='age_text', readonly=True, null=True)
-    icon = fields.CharField(attribute='icon', readonly=True)
+    Meta = generate_meta(
+        Device,
+        {
+            "cache": HttpCache(settings.DEVICE_HTTP_CACHE_TIMEOUT),
+            "serializer": CSVSerializer(),
+        },
+    )
+    age_minutes = fields.IntegerField(attribute="age_minutes", readonly=True, null=True)
+    age_colour = fields.CharField(attribute="age_colour", readonly=True, null=True)
+    age_text = fields.CharField(attribute="age_text", readonly=True, null=True)
+    icon = fields.CharField(attribute="icon", readonly=True)