Merge branch 'ocr-bulk-import'

.pre-commit-config.yaml

@@ -10,13 +10,13 @@ repos:
       - id: check-yaml
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.17.0
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
 
   - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.8.0
     hooks:
       - id: black
 
@@ -27,7 +27,7 @@ repos:
         args: ["--profile", "black"]
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         args: ["--config=setup.cfg","--per-file-ignores=boranga/settings.py:F405,E402","--ignore=E203, W503"]

Dockerfile

@@ -85,16 +85,10 @@ RUN chmod 755 /startup.sh && \
     mkdir /app && \
     chown -R oim.oim /app && \
     ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
-    wget https://raw.githubusercontent.com/dbca-wa/wagov_utils/main/wagov_utils/bin/health_check.sh -O /bin/health_check.sh && \
-    chmod 755 /bin/health_check.sh && \
-    wget https://raw.githubusercontent.com/dbca-wa/wagov_utils/main/wagov_utils/bin-python/scheduler/scheduler.py -O /bin/scheduler.py && \
-    chmod 755 /bin/scheduler.py && \
-    mkdir /tmp/azcopy/ && \
-    wget https://aka.ms/downloadazcopy-v10-linux -O /tmp/azcopy/azcopy.tar.gz && \
-    cd /tmp/azcopy/ ; tar -xzvf azcopy.tar.gz && \
-    cp /tmp/azcopy/azcopy_linux_amd64_10.*/azcopy /bin/azcopy && \
-    chmod 755 /bin/azcopy && \
-    rm -rf /tmp/azcopy/
+    wget https://raw.githubusercontent.com/dbca-wa/wagov_utils/main/wagov_utils/bin/default_script_installer.sh -O /tmp/default_script_installer.sh && \
+    chmod 755 /tmp/default_script_installer.sh && \
+    /tmp/default_script_installer.sh && \
+    rm -rf /tmp/*
 
 FROM configure_boranga as python_dependencies_boranga
 

boranga/components/main/api.py

@@ -3,16 +3,21 @@
 
 import pyproj
 from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
 from django.core.cache import cache
+from django_filters import rest_framework as filters
+from rest_framework import filters as rest_framework_filters
 from rest_framework import viewsets
 
 from boranga import helpers
 from boranga.components.main.models import GlobalSettings, HelpTextEntry
 from boranga.components.main.serializers import (
+    ContentTypeSerializer,
     GlobalSettingsSerializer,
     HelpTextEntrySerializer,
 )
 from boranga.components.occurrence.models import Datum
+from boranga.permissions import IsInternal
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +46,15 @@ def get_queryset(self):
         return qs
 
 
+class ContentTypeViewSet(viewsets.ReadOnlyModelViewSet):
+    queryset = ContentType.objects.filter(app_label="boranga")
+    serializer_class = ContentTypeSerializer
+    permission_classes = [IsInternal]
+    filter_backends = [filters.DjangoFilterBackend, rest_framework_filters.SearchFilter]
+    filterset_fields = ["app_label", "model"]
+    search_fields = ["^model"]
+
+
 class RetrieveActionLoggingViewsetMixin:
     """Mixin to automatically log user actions when a user retrieves an instance.
 

boranga/components/main/serializers.py

@@ -1,3 +1,7 @@
+import logging
+
+from django.contrib.contenttypes.models import ContentType
+from django.db.models.fields.related import ForeignKey, OneToOneField
 from ledger_api_client.ledger_models import EmailUserRO
 from ledger_api_client.ledger_models import EmailUserRO as EmailUser
 from rest_framework import serializers
@@ -7,7 +11,12 @@
     GlobalSettings,
     HelpTextEntry,
 )
-from boranga.helpers import is_django_admin
+from boranga.helpers import (
+    get_openpyxl_data_validation_type_for_django_field,
+    is_django_admin,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class CommunicationLogEntrySerializer(serializers.ModelSerializer):
@@ -109,3 +118,60 @@ class Meta:
 
     def get_user_can_administer(self, obj):
         return is_django_admin(self.context["request"])
+
+
+class ContentTypeSerializer(serializers.ModelSerializer):
+    model_fields = serializers.SerializerMethodField()
+    model_verbose_name = serializers.SerializerMethodField()
+
+    class Meta:
+        model = ContentType
+        fields = "__all__"
+
+    def get_model_verbose_name(self, obj):
+        if not obj.model_class():
+            return None
+        return obj.model_class()._meta.verbose_name.title()
+
+    def get_model_fields(self, obj):
+        if not obj.model_class():
+            return []
+        fields = obj.model_class()._meta.get_fields()
+
+        def filter_fields(field):
+            return not field.auto_created and not (
+                field.is_relation
+                and type(field)
+                not in [
+                    ForeignKey,
+                    OneToOneField,
+                ]
+            )
+
+        fields = list(filter(filter_fields, fields))
+        model_fields = []
+        for field in fields:
+            display_name = (
+                field.verbose_name.title()
+                if hasattr(field, "verbose_name")
+                else field.name
+            )
+            field_type = str(type(field)).split(".")[-1].replace("'>", "")
+            choices = field.choices if hasattr(field, "choices") else None
+            allow_null = field.null if hasattr(field, "null") else None
+            max_length = field.max_length if hasattr(field, "max_length") else None
+            xlsx_validation_type = get_openpyxl_data_validation_type_for_django_field(
+                field
+            )
+            model_fields.append(
+                {
+                    "name": field.name,
+                    "display_name": display_name,
+                    "type": field_type,
+                    "choices": choices,
+                    "allow_null": allow_null,
+                    "max_length": max_length,
+                    "xlsx_validation_type": xlsx_validation_type,
+                }
+            )
+        return model_fields

boranga/components/occurrence/admin.py

@@ -16,6 +16,9 @@
     LocationAccuracy,
     ObservationMethod,
     OccurrenceGeometry,
+    OccurrenceReportBulkImportSchema,
+    OccurrenceReportBulkImportSchemaColumn,
+    OccurrenceReportBulkImportTask,
     OccurrenceReportGeometry,
     OccurrenceSite,
     OccurrenceTenure,
@@ -466,6 +469,25 @@ class CountedSubjectAdmin(ArchivableModelAdminMixin, DeleteProtectedModelAdmin):
     list_display = ["name"]
 
 
+class OccurrenceReportBulkImportTaskAdmin(DeleteProtectedModelAdmin):
+    list_display = ["id", "datetime_queued", "processing_status"]
+    list_filter = ["processing_status", "datetime_completed"]
+    readonly_fields = ["datetime_queued"]
+
+
+class OccurrenceReportBulkImportSchemaColumnInline(admin.StackedInline):
+    model = OccurrenceReportBulkImportSchemaColumn
+    extra = 0
+
+
+class OccurrenceReportBulkImportSchemaAdmin(DeleteProtectedModelAdmin):
+    list_display = ["group_type", "version", "datetime_created", "datetime_updated"]
+    readonly_fields = ["datetime_created", "datetime_updated"]
+    list_filter = ["group_type"]
+    inlines = [OccurrenceReportBulkImportSchemaColumnInline]
+    ordering = ["group_type", "version"]
+
+
 # Each of the following models will be available to Django Admin.
 admin.site.register(LandForm, LandFormAdmin)
 admin.site.register(RockType, RockTypeAdmin)
@@ -494,3 +516,7 @@ class CountedSubjectAdmin(ArchivableModelAdminMixin, DeleteProtectedModelAdmin):
 admin.site.register(LocationAccuracy, LocationAccuracyAdmin)
 admin.site.register(WildStatus, WildStatusAdmin)
 admin.site.register(OccurrenceSite)
+admin.site.register(OccurrenceReportBulkImportTask, OccurrenceReportBulkImportTaskAdmin)
+admin.site.register(
+    OccurrenceReportBulkImportSchema, OccurrenceReportBulkImportSchemaAdmin
+)

boranga/components/occurrence/api.py

@@ -12,6 +12,7 @@
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404, redirect
 from django.urls import reverse
+from django_filters import rest_framework as filters
 from ledger_api_client.ledger_models import EmailUserRO as EmailUser
 from multiselectfield import MultiSelectField
 from openpyxl import Workbook
@@ -21,6 +22,7 @@
 from rest_framework.decorators import action as detail_route
 from rest_framework.decorators import action as list_route
 from rest_framework.decorators import renderer_classes
+from rest_framework.pagination import LimitOffsetPagination
 from rest_framework.permissions import AllowAny
 from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
@@ -65,6 +67,8 @@
     OccurrenceReport,
     OccurrenceReportAmendmentRequest,
     OccurrenceReportAmendmentRequestDocument,
+    OccurrenceReportBulkImportSchema,
+    OccurrenceReportBulkImportTask,
     OccurrenceReportDocument,
     OccurrenceReportGeometry,
     OccurrenceReportReferral,
@@ -110,6 +114,7 @@
     IsOccurrenceReportReferee,
     OccurrenceObjectPermission,
     OccurrencePermission,
+    OccurrenceReportBulkImportPermission,
     OccurrenceReportCopyPermission,
     OccurrenceReportObjectPermission,
     OccurrenceReportPermission,
@@ -132,6 +137,9 @@
     OccurrenceDocumentSerializer,
     OccurrenceLogEntrySerializer,
     OccurrenceReportAmendmentRequestSerializer,
+    OccurrenceReportBulkImportSchemaColumnSerializer,
+    OccurrenceReportBulkImportSchemaSerializer,
+    OccurrenceReportBulkImportTaskSerializer,
     OccurrenceReportDocumentSerializer,
     OccurrenceReportLogEntrySerializer,
     OccurrenceReportProposalReferralSerializer,
@@ -6230,3 +6238,112 @@ def retract(self, request, *args, **kwargs):
             instance.occurrence_report, context={"request": request}
         )
         return Response(serializer.data, status=status.HTTP_200_OK)
+
+
+class OccurrenceReportBulkImportTaskViewSet(
+    viewsets.GenericViewSet,
+    mixins.CreateModelMixin,
+    mixins.ListModelMixin,
+):
+    queryset = OccurrenceReportBulkImportTask.objects.all()
+    permission_classes = [OccurrenceReportBulkImportPermission]
+    serializer_class = OccurrenceReportBulkImportTaskSerializer
+    filter_backends = [filters.DjangoFilterBackend]
+    filterset_fields = ["processing_status"]
+    pagination_class = LimitOffsetPagination
+
+    def perform_create(self, serializer):
+        serializer.save(email_user=self.request.user.id)
+
+    @detail_route(methods=["patch"], detail=True)
+    def retry(self, request, *args, **kwargs):
+        instance = self.get_object()
+        instance.retry()
+        return Response(status=status.HTTP_200_OK)
+
+    @detail_route(methods=["patch"], detail=True)
+    def revert(self, request, *args, **kwargs):
+        instance = self.get_object()
+        instance.revert()
+        return Response(status=status.HTTP_200_OK)
+
+
+class OccurrenceReportBulkImportSchemaViewSet(
+    viewsets.GenericViewSet,
+    mixins.RetrieveModelMixin,
+    mixins.CreateModelMixin,
+    mixins.UpdateModelMixin,
+    mixins.ListModelMixin,
+):
+    queryset = OccurrenceReportBulkImportSchema.objects.all()
+    serializer_class = OccurrenceReportBulkImportSchemaSerializer
+    permission_classes = [OccurrenceReportBulkImportPermission]
+    filter_backends = [filters.DjangoFilterBackend]
+    filterset_fields = ["group_type"]
+
+    def get_queryset(self):
+        qs = self.queryset
+        if not (is_internal(self.request) or self.request.user.is_superuser):
+            qs = OccurrenceReportBulkImportSchema.objects.none()
+        return qs
+
+    def perform_create(self, serializer):
+        latest_version = (
+            OccurrenceReportBulkImportSchema.objects.filter(
+                group_type=serializer.validated_data["group_type"]
+            )
+            .order_by("-version")
+            .first()
+            .version
+        )
+        serializer.save(version=latest_version + 1)
+        return super().perform_create(serializer)
+
+    @list_route(methods=["get"], detail=False)
+    def get_schema_list_by_group_type(self, request, *args, **kwargs):
+        group_type = request.GET.get("group_type", None)
+        if not group_type:
+            raise serializers.ValidationError(
+                "Group Type is required to return correct list of values"
+            )
+
+        group_type = GroupType.objects.get(name=group_type)
+
+        schema = OccurrenceReportBulkImportSchema.objects.filter(group_type=group_type)
+        serializer = OccurrenceReportBulkImportSchemaSerializer(schema, many=True)
+        return Response(serializer.data)
+
+    @detail_route(methods=["get"], detail=True)
+    def preview_import_file(self, request, *args, **kwargs):
+        instance = self.get_object()
+        buffer = BytesIO()
+        workbook = instance.preview_import_file
+        workbook.save(buffer)
+        buffer.seek(0)
+        filename = f"bulk-import-schema-{instance.group_type.name}-version-{instance.version}-preview.xlsx"
+        response = HttpResponse(buffer.read(), content_type="application/vnd.ms-excel")
+        response["Content-Disposition"] = f"attachment; filename={filename}"
+        buffer.close()
+        return response
+
+    @detail_route(methods=["post"], detail=True)
+    def copy(self, request, *args, **kwargs):
+        instance = self.get_object()
+        new_instance = instance.copy()
+        serializer = OccurrenceReportBulkImportSchemaSerializer(new_instance)
+        return Response(serializer.data, status=status.HTTP_201_CREATED)
+
+    @detail_route(methods=["put"], detail=True)
+    def save_column(self, request, *args, **kwargs):
+        instance = self.get_object()
+        column_data = request.data.get("column_data", None)
+        if not column_data:
+            raise serializers.ValidationError("Column data is required")
+        serializer = OccurrenceReportBulkImportSchemaColumnSerializer(
+            instance, data=column_data
+        )
+        serializer.is_valid(raise_exception=True)
+        serializer.save()
+
+        serializer = OccurrenceReportBulkImportSchemaSerializer(instance)
+        return Response(serializer.data, status=status.HTTP_201_CREATED)