Merge branch 'ocr-bulk-import'

boranga/components/main/api.py

@@ -54,13 +54,29 @@ class ContentTypeViewSet(viewsets.ReadOnlyModelViewSet):
     )
     def ocr_bulk_import_content_types(self, request):
         """Returns a list of content types that are allowed to be imported in the ocr bulk importer"""
-        content_types = ContentType.objects.filter(
-            app_label="boranga",
-        ).filter(
-            Q(model__startswith="occurrencereport")
-            | Q(model__startswith="ocr")
-            | Q(model__iexact="occurrence")
-            | Q(model__iexact="submitterinformation")
+        content_types = (
+            ContentType.objects.filter(
+                app_label="boranga",
+            )
+            .filter(
+                Q(model__startswith="occurrencereport")
+                | Q(model__startswith="ocr")
+                | Q(model__iexact="occurrence")
+                | Q(model__iexact="submitterinformation")
+            )
+            .exclude(
+                model__in=[
+                    "occurrencereportproposalrequest",
+                    "occurrencereportdeclineddetails",
+                    "occurrencereportshapefiledocument",
+                ]
+            )
+            .exclude(model__icontains="amendment")
+            .exclude(model__icontains="bulkimport")
+            .exclude(model__icontains="referral")
+            .exclude(model__icontains="referee")
+            .exclude(model__icontains="occurrencereportlog")
+            .exclude(model__icontains="useraction")
         )
         serializer = self.get_serializer(content_types, many=True)
         return Response(serializer.data)

boranga/components/main/serializers.py

@@ -117,6 +117,7 @@ def get_user_can_administer(self, obj):
 class ContentTypeSerializer(serializers.ModelSerializer):
     model_fields = serializers.SerializerMethodField()
     model_verbose_name = serializers.SerializerMethodField()
+    model_abbreviation = serializers.SerializerMethodField()
 
     class Meta:
         model = ContentType
@@ -127,6 +128,11 @@ def get_model_verbose_name(self, obj):
             return None
         return obj.model_class()._meta.verbose_name.title()
 
+    def get_model_abbreviation(self, obj):
+        if not obj.model_class():
+            return None
+        return obj.model_class().BULK_IMPORT_ABBREVIATION
+
     def get_model_fields(self, obj):
         if not obj.model_class():
             return []

boranga/components/occurrence/models.py

@@ -31,7 +31,16 @@
 from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import IntegrityError, models, transaction
-from django.db.models import CharField, Count, Func, ManyToManyField, Max, Q
+from django.db.models import (
+    CharField,
+    Count,
+    Func,
+    ManyToManyField,
+    Max,
+    OuterRef,
+    Q,
+    Subquery,
+)
 from django.db.models.functions import Cast, Length
 from django.utils import timezone
 from django.utils.functional import cached_property
@@ -150,6 +159,7 @@ class OccurrenceReport(SubmitterInformationModelMixin, RevisionedMixin):
 
     objects = OccurrenceReportManager()
 
+    BULK_IMPORT_ABBREVIATION = "ocr"
     BULK_IMPORT_EXCLUDE_FIELDS = ["occurrence_report_number", "import_hash"]
 
     CUSTOMER_STATUS_DRAFT = "draft"
@@ -1331,6 +1341,8 @@ class Meta:
 
 
 class OccurrenceReportApprovalDetails(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrapp"
+
     occurrence_report = models.OneToOneField(
         OccurrenceReport, on_delete=models.CASCADE, related_name="approval_details"
     )
@@ -1904,6 +1916,8 @@ def __str__(self):
 
 # NOTE: this and OCCLocation have a number of unused fields that should be removed
 class OCRLocation(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrloc"
+
     """
     Location data  for occurrence report
 
@@ -2125,6 +2139,8 @@ class Meta:
 
 
 class OccurrenceReportGeometry(GeometryBase, DrawnByGeometry):
+    BULK_IMPORT_ABBREVIATION = "ocrgeo"
+
     occurrence_report = models.ForeignKey(
         OccurrenceReport,
         on_delete=models.CASCADE,
@@ -2154,6 +2170,8 @@ def save(self, *args, **kwargs):
 
 
 class OCRObserverDetail(RevisionedMixin):
+    BULK_IMPORT_ABBREVIATION = "ocrcon"
+
     """
     Observer data  for occurrence report
 
@@ -2347,6 +2365,7 @@ def __str__(self):
 
 
 class OCRHabitatComposition(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrhab"
     """
     Habitat data  for occurrence report
 
@@ -2400,6 +2419,7 @@ def __init__(self, *args, **kwargs):
 
 
 class OCRHabitatCondition(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrhq"
     """
     Habitat Condition data for occurrence report
 
@@ -2461,6 +2481,8 @@ def __str__(self):
 
 
 class OCRVegetationStructure(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrveg"
+
     """
     Vegetation Structure data for occurrence report
 
@@ -2517,6 +2539,8 @@ def __str__(self):
 
 
 class OCRFireHistory(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrfh"
+
     """
     Fire History data for occurrence report
 
@@ -2546,6 +2570,8 @@ def __str__(self):
 
 
 class OCRAssociatedSpecies(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrspe"
+
     """
     Associated Species data for occurrence report
 
@@ -2600,6 +2626,8 @@ def __str__(self):
 
 
 class OCRObservationDetail(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrobs"
+
     """
     Observation Details data for occurrence report
 
@@ -2737,6 +2765,8 @@ def __str__(self):
 
 
 class OCRPlantCount(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrnum"
+
     """
     Plant Count data for occurrence report
 
@@ -2943,6 +2973,8 @@ def __str__(self):
 
 
 class OCRAnimalObservation(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrnum"
+
     """
     Animal Observation data for occurrence report
 
@@ -3130,6 +3162,8 @@ def __str__(self):
 
 
 class OCRIdentification(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrid"
+
     """
     Identification data for occurrence report
 
@@ -3171,6 +3205,8 @@ def __str__(self):
 
 
 class OccurrenceReportDocument(Document):
+    BULK_IMPORT_ABBREVIATION = "ocrdoc"
+
     document_number = models.CharField(max_length=9, blank=True, default="")
     occurrence_report = models.ForeignKey(
         "OccurrenceReport", related_name="documents", on_delete=models.CASCADE
@@ -3284,6 +3320,7 @@ class Meta:
 
 
 class OCRConservationThreat(RevisionedMixin):
+    BULK_IMPORT_ABBREVIATION = "ocrthr"
     """
     Threat for a occurrence_report in a particular location.
 
@@ -3381,6 +3418,7 @@ def get_queryset(self):
 
 
 class Occurrence(RevisionedMixin):
+    BULK_IMPORT_ABBREVIATION = "occ"
 
     REVIEW_STATUS_CHOICES = (
         ("not_reviewed", "Not Reviewed"),
@@ -6737,11 +6775,6 @@ def related_model_qs(self):
         if issubclass(self.related_model, ArchivableModel):
             related_model_qs = self.related_model.objects.exclude(archived=True)
 
-        if hasattr(self.related_model, "group_type"):
-            related_model_qs = related_model_qs.only(display_field, "group_type")
-        else:
-            related_model_qs = related_model_qs.only(display_field)
-
         return related_model_qs.order_by(display_field)
 
     @cached_property
@@ -6872,11 +6905,34 @@ def get_sample_value(self, errors, species_or_community_identifier=None):
         if isinstance(field, models.ForeignKey):
             related_model_qs = self.filtered_related_model_qs
 
+            # Special case for species or community
+            # Ensure only species or communities that have occurrences are selected
+            # in case the schema includes the occurrence model (quite likely)
+            if field.name == "species":
+                related_model_qs = related_model_qs.annotate(
+                    occurrence_count=Subquery(
+                        Occurrence.objects.filter(species__pk=OuterRef("pk"))
+                        .values("species")
+                        .annotate(count=Count("id"))
+                        .values("count")
+                    )
+                ).filter(occurrence_count__gt=0)
+            if field.name == "community":
+                related_model_qs = related_model_qs.annotate(
+                    occurrence_count=Subquery(
+                        Occurrence.objects.filter(community__pk=OuterRef("pk"))
+                        .values("community")
+                        .annotate(count=Count("id"))
+                        .values("count")
+                    )
+                ).filter(occurrence_count__gt=0)
+
             if not related_model_qs.exists():
+                error_message = f"No records found for foreign key field {field.related_model._meta.model_name}"
                 errors.append(
                     {
                         "error_type": "no_records",
-                        "error_message": f"No records found for foreign key {field.related_model._meta.model_name}",
+                        "error_message": error_message,
                     }
                 )
 
@@ -6987,6 +7043,18 @@ def get_sample_value(self, errors, species_or_community_identifier=None):
                     filter_field = {
                         "community__taxonomy__community_migrated_id": species_or_community_identifier
                     }
+                if not random_occurrence.filter(**filter_field).exists():
+                    error_message = (
+                        f"No occurrences found where species or community identifier = "
+                        f"{species_or_community_identifier}"
+                    )
+                    errors.append(
+                        {
+                            "error_type": "no_occurrences",
+                            "error_message": error_message,
+                        }
+                    )
+                    return None
                 return (
                     random_occurrence.filter(**filter_field)
                     .order_by("?")

boranga/components/users/models.py

@@ -52,6 +52,8 @@ def get_filter_list(cls):
 
 
 class SubmitterInformation(models.Model):
+    BULK_IMPORT_ABBREVIATION = "ocrsub"
+
     email_user = models.IntegerField(blank=True, null=True)
     name = models.CharField(max_length=100, blank=True, null=True)
     contact_details = models.TextField(blank=True, null=True)