From 6f2f00930ffaae8abf5f9dbfc7129a732f249525 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 5 Sep 2024 14:27:20 +0800 Subject: [PATCH 01/29] Remove unused document field from OccurrenceReportReferral. --- boranga/components/occurrence/models.py | 7 ------- ...ccurrencereportreferral_document_and_more.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 boranga/migrations/0449_remove_occurrencereportreferral_document_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 833bc4cc..8d9a0808 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1631,13 +1631,6 @@ class OccurrenceReportReferral(models.Model): blank=True ) # used in other projects for complete referral comment but not used in boranga referral_comment = models.TextField(blank=True, null=True) # Referral Comment - document = models.ForeignKey( - OccurrenceReportReferralDocument, - blank=True, - null=True, - related_name="referral_document", - on_delete=models.SET_NULL, - ) assigned_officer = models.IntegerField(null=True) # EmailUserRO is_external = models.BooleanField(default=False) diff --git a/boranga/migrations/0449_remove_occurrencereportreferral_document_and_more.py b/boranga/migrations/0449_remove_occurrencereportreferral_document_and_more.py new file mode 100644 index 00000000..641c347d --- /dev/null +++ b/boranga/migrations/0449_remove_occurrencereportreferral_document_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.9 on 2024-09-05 06:26 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0448_remove_communitytaxonomy_name_currency_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="occurrencereportreferral", + name="document", + ), + ] From 50b1e580b3f83a13b72241ff0b6f02020f047252 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 5 Sep 2024 14:28:52 +0800 Subject: [PATCH 02/29] Remove unused model OccurrenceReportReferralDocument --- boranga/components/occurrence/models.py | 31 ------------------- ..._alter_communitydocument__file_and_more.py | 16 ++++++++++ 2 files changed, 16 insertions(+), 31 deletions(-) create mode 100644 boranga/migrations/0450_alter_communitydocument__file_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 8d9a0808..7f76e604 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1563,37 +1563,6 @@ def delete(self): return super().delete() -class OccurrenceReportReferralDocument(Document): - referral = models.ForeignKey( - "OccurrenceReportReferral", - related_name="referral_documents", - on_delete=models.CASCADE, - ) - _file = models.FileField( - upload_to=update_occurrence_report_referral_doc_filename, - max_length=512, - storage=private_storage, - ) - input_name = models.CharField(max_length=255, null=True, blank=True) - can_delete = models.BooleanField( - default=True - ) # after initial submit prevent document from being deleted - - def delete(self): - if self.can_delete: - if self._file: - self._file.delete() - return super().delete() - logger.info( - "Cannot delete existing document object after occurrence report referral has been submitted: {}".format( - self.name - ) - ) - - class Meta: - app_label = "boranga" - - class OccurrenceReportReferral(models.Model): SENT_CHOICE_FROM_ASSESSOR = 1 SENT_CHOICE_FROM_REFERRAL = 2 diff --git a/boranga/migrations/0450_alter_communitydocument__file_and_more.py b/boranga/migrations/0450_alter_communitydocument__file_and_more.py new file mode 100644 index 00000000..0abc6de3 --- /dev/null +++ b/boranga/migrations/0450_alter_communitydocument__file_and_more.py @@ -0,0 +1,16 @@ +# Generated by Django 5.0.9 on 2024-09-05 06:27 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0449_remove_occurrencereportreferral_document_and_more"), + ] + + operations = [ + migrations.DeleteModel( + name="OccurrenceReportReferralDocument", + ), + ] From 2f6ff56697154cda852929d90d07bc95175e926b Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 5 Sep 2024 14:41:01 +0800 Subject: [PATCH 03/29] Bug fix: In the case of completing a referral, the geometry_data is already a dict so added code to be able to cope with such a scenario. --- boranga/components/spatial/utils.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/boranga/components/spatial/utils.py b/boranga/components/spatial/utils.py index af728588..74fd8c2f 100644 --- a/boranga/components/spatial/utils.py +++ b/boranga/components/spatial/utils.py @@ -4,13 +4,13 @@ import re import sys import urllib.parse +import xml.etree.ElementTree as ET from itertools import combinations import geojson import numpy as np import requests import shapely.geometry as shp -import xml.etree.ElementTree as ET from django.apps import apps from django.contrib.contenttypes import models as ct_models from django.contrib.gis.geos import GEOSGeometry @@ -73,7 +73,8 @@ def intersect_geometry_with_layer( # e.g. MULTIPOINT (3 1, 4 1, 5 2), and rather throws unintelligible java class exceptions at me, so we # have to convert them to a double-bracket notation in the form of MULTIPOINT ((3 1), (4 1), (5 2)). Even # though both forms are (topologically) valid by OGC definition, the jts (java topology suite) library only - # seems to except singleton lists (https://www.tsusiatsoftware.net/jts/javadoc/com/vividsolutions/jts/io/WKTReader.html) + # seems to except singleton lists + # (https://www.tsusiatsoftware.net/jts/javadoc/com/vividsolutions/jts/io/WKTReader.html) logger.warn( f"Converting MultiPoint geometry {test_geom} to double-bracket notation" ) @@ -271,8 +272,11 @@ def save_geometry( ) if instance_fk_field_name is None: instance_fk_field_name = instance_model_name.lower() + if isinstance(geometry_data, dict): + geometry = geometry_data + else: + geometry = json.loads(geometry_data) - geometry = json.loads(geometry_data) if ( 0 == len(geometry["features"]) and 0 @@ -421,11 +425,13 @@ def save_geometry( if number_matched: if error_value and number_matched >= error_value: logger.info( - f"Rejecting geometry {geom[0]}, it intersects with {number_matched} features from {intersect_layer.layer_name}. " + f"Rejecting geometry {geom[0]}, it intersects with {number_matched} " + f"features from {intersect_layer.layer_name}. " f"Error value: {error_value}" ) raise serializers.ValidationError( - f"Geometry intersects with too many features from {intersect_layer.layer_name}: {number_matched}. Error value: {error_value}" + f"Geometry intersects with too many features from " + f"{intersect_layer.layer_name}: {number_matched}. Error value: {error_value}" ) intersect_data = intersect_geometry_with_layer( From c8c7e181db1275f0e01e1d4885594b54ac2b69ca Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 5 Sep 2024 14:49:33 +0800 Subject: [PATCH 04/29] Remove unused field referral_text from OccurrenceReportReferral model. --- boranga/components/occurrence/models.py | 3 --- ...encereportreferral_referral_text_and_more.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 boranga/migrations/0451_remove_occurrencereportreferral_referral_text_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 7f76e604..1bdf2069 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1596,9 +1596,6 @@ class OccurrenceReportReferral(models.Model): default=PROCESSING_STATUS_CHOICES[0][0], ) text = models.TextField(blank=True) # Assessor text when send_referral - referral_text = models.TextField( - blank=True - ) # used in other projects for complete referral comment but not used in boranga referral_comment = models.TextField(blank=True, null=True) # Referral Comment assigned_officer = models.IntegerField(null=True) # EmailUserRO is_external = models.BooleanField(default=False) diff --git a/boranga/migrations/0451_remove_occurrencereportreferral_referral_text_and_more.py b/boranga/migrations/0451_remove_occurrencereportreferral_referral_text_and_more.py new file mode 100644 index 00000000..f4d8dfde --- /dev/null +++ b/boranga/migrations/0451_remove_occurrencereportreferral_referral_text_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.9 on 2024-09-05 06:47 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0450_alter_communitydocument__file_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="occurrencereportreferral", + name="referral_text", + ), + ] From 101d4fb57f748024da4107b1cd6425b618235da7 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 5 Sep 2024 14:56:41 +0800 Subject: [PATCH 05/29] Remove unused sent_from field from OccurrenceReportReferral model. --- boranga/components/occurrence/models.py | 12 ------------ ...currencereportreferral_sent_from_and_more.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 boranga/migrations/0452_remove_occurrencereportreferral_sent_from_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 1bdf2069..5adfe7b2 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1095,20 +1095,11 @@ def send_referral(self, request, referral_email, referral_text): ).exists(): raise ValidationError("A referral has already been sent to this user") - # Check if the user sending the referral is a referee themselves - sent_from = OccurrenceReportReferral.SENT_CHOICE_FROM_ASSESSOR - if OccurrenceReportReferral.objects.filter( - occurrence_report=self, - referral=request.user.id, - ).exists(): - sent_from = OccurrenceReportReferral.SENT_CHOICE_FROM_REFERRAL - # Create Referral referral = OccurrenceReportReferral.objects.create( occurrence_report=self, referral=referee.id, sent_by=request.user.id, - sent_from=sent_from, text=referral_text, assigned_officer=request.user.id, ) @@ -1586,9 +1577,6 @@ class OccurrenceReportReferral(models.Model): sent_by = models.IntegerField() # EmailUserRO referral = models.IntegerField() # EmailUserRO linked = models.BooleanField(default=False) - sent_from = models.SmallIntegerField( - choices=SENT_CHOICES, default=SENT_CHOICES[0][0] - ) processing_status = models.CharField( "Processing Status", max_length=30, diff --git a/boranga/migrations/0452_remove_occurrencereportreferral_sent_from_and_more.py b/boranga/migrations/0452_remove_occurrencereportreferral_sent_from_and_more.py new file mode 100644 index 00000000..ddda98a8 --- /dev/null +++ b/boranga/migrations/0452_remove_occurrencereportreferral_sent_from_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.9 on 2024-09-05 06:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0451_remove_occurrencereportreferral_referral_text_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="occurrencereportreferral", + name="sent_from", + ), + ] From 9ccea6a99d3f5163c19068329abee64b5ceecd87 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 6 Sep 2024 09:52:18 +0800 Subject: [PATCH 06/29] Remove unused boundary field from OCRLocation. --- boranga/components/occurrence/models.py | 1 - ...0453_remove_ocrlocation_boundary_and_more.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 boranga/migrations/0453_remove_ocrlocation_boundary_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 5adfe7b2..ee432999 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1875,7 +1875,6 @@ class OCRLocation(models.Model): location_description = models.TextField(null=True, blank=True) boundary_description = models.TextField(null=True, blank=True) new_occurrence = models.BooleanField(null=True, blank=True) - boundary = models.IntegerField(null=True, blank=True, default=0) mapped_boundary = models.BooleanField(null=True, blank=True) buffer_radius = models.IntegerField(null=True, blank=True, default=0) datum = models.ForeignKey(Datum, on_delete=models.SET_NULL, null=True, blank=True) diff --git a/boranga/migrations/0453_remove_ocrlocation_boundary_and_more.py b/boranga/migrations/0453_remove_ocrlocation_boundary_and_more.py new file mode 100644 index 00000000..33e68382 --- /dev/null +++ b/boranga/migrations/0453_remove_ocrlocation_boundary_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.9 on 2024-09-06 01:51 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0452_remove_occurrencereportreferral_sent_from_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="ocrlocation", + name="boundary", + ), + ] From f1ceda0b0fa0524141a0059aaab449d9dc197643 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 6 Sep 2024 09:56:06 +0800 Subject: [PATCH 07/29] Remove unused new_occurrence field from OCRLocation model. --- boranga/components/occurrence/models.py | 1 - ...emove_ocrlocation_new_occurrence_and_more.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 boranga/migrations/0454_remove_ocrlocation_new_occurrence_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index ee432999..a473374f 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1874,7 +1874,6 @@ class OCRLocation(models.Model): ) location_description = models.TextField(null=True, blank=True) boundary_description = models.TextField(null=True, blank=True) - new_occurrence = models.BooleanField(null=True, blank=True) mapped_boundary = models.BooleanField(null=True, blank=True) buffer_radius = models.IntegerField(null=True, blank=True, default=0) datum = models.ForeignKey(Datum, on_delete=models.SET_NULL, null=True, blank=True) diff --git a/boranga/migrations/0454_remove_ocrlocation_new_occurrence_and_more.py b/boranga/migrations/0454_remove_ocrlocation_new_occurrence_and_more.py new file mode 100644 index 00000000..32a98791 --- /dev/null +++ b/boranga/migrations/0454_remove_ocrlocation_new_occurrence_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.9 on 2024-09-06 01:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0453_remove_ocrlocation_boundary_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="ocrlocation", + name="new_occurrence", + ), + ] From d8d857bef6b24ea9d6c44f52cbbdc3de7e1af747 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 6 Sep 2024 09:58:51 +0800 Subject: [PATCH 08/29] Remove unused intersects field from OccurrenceReportGeometry and OccurrenceGeometry models. --- boranga/components/occurrence/models.py | 11 ++-------- ..._occurrencegeometry_intersects_and_more.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 boranga/migrations/0455_remove_occurrencegeometry_intersects_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index a473374f..0666ea1a 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -2082,14 +2082,7 @@ class Meta: abstract = True -class IntersectsGeometry(models.Model): - intersects = models.BooleanField(default=False) - - class Meta: - abstract = True - - -class OccurrenceReportGeometry(GeometryBase, DrawnByGeometry, IntersectsGeometry): +class OccurrenceReportGeometry(GeometryBase, DrawnByGeometry): occurrence_report = models.ForeignKey( OccurrenceReport, on_delete=models.CASCADE, @@ -4219,7 +4212,7 @@ class GeometryType(Func): output_field = CharField() -class OccurrenceGeometry(GeometryBase, DrawnByGeometry, IntersectsGeometry): +class OccurrenceGeometry(GeometryBase, DrawnByGeometry): occurrence = models.ForeignKey( Occurrence, on_delete=models.CASCADE, diff --git a/boranga/migrations/0455_remove_occurrencegeometry_intersects_and_more.py b/boranga/migrations/0455_remove_occurrencegeometry_intersects_and_more.py new file mode 100644 index 00000000..66ec5c0e --- /dev/null +++ b/boranga/migrations/0455_remove_occurrencegeometry_intersects_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 5.0.9 on 2024-09-06 01:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0454_remove_ocrlocation_new_occurrence_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="occurrencegeometry", + name="intersects", + ), + migrations.RemoveField( + model_name="occurrencereportgeometry", + name="intersects", + ), + ] From cdfe8b1b17d24d4841bcec8f6a3d9e072aa0e795 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 6 Sep 2024 10:11:24 +0800 Subject: [PATCH 09/29] Remove unused boundary field from OCCLocation model. --- boranga/components/occurrence/models.py | 2 -- ...0456_remove_occlocation_boundary_and_more.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 boranga/migrations/0456_remove_occlocation_boundary_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 0666ea1a..7ca97a3f 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -4177,8 +4177,6 @@ class OCCLocation(models.Model): ) location_description = models.TextField(null=True, blank=True) boundary_description = models.TextField(null=True, blank=True) - - boundary = models.IntegerField(null=True, blank=True, default=0) mapped_boundary = models.BooleanField(null=True, blank=True) buffer_radius = models.IntegerField(null=True, blank=True, default=0) datum = models.ForeignKey(Datum, on_delete=models.SET_NULL, null=True, blank=True) diff --git a/boranga/migrations/0456_remove_occlocation_boundary_and_more.py b/boranga/migrations/0456_remove_occlocation_boundary_and_more.py new file mode 100644 index 00000000..6f1d2422 --- /dev/null +++ b/boranga/migrations/0456_remove_occlocation_boundary_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 5.0.9 on 2024-09-06 02:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0455_remove_occurrencegeometry_intersects_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="occlocation", + name="boundary", + ), + ] From 2470fba657c2c33632092a3129c5af2374868ca6 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 6 Sep 2024 10:12:10 +0800 Subject: [PATCH 10/29] Remove references to fields that have been removed. --- boranga/components/occurrence/serializers.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index d5bf7cca..12494f1c 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -758,9 +758,6 @@ class Meta: class OCRLocationSerializer(serializers.ModelSerializer): - # observation_date = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S") - # geojson_point = serializers.SerializerMethodField() - # geojson_polygon = serializers.SerializerMethodField() has_boundary = serializers.SerializerMethodField() has_points = serializers.SerializerMethodField() coordinate_source = serializers.CharField( @@ -775,11 +772,8 @@ class Meta: fields = ( "id", "occurrence_report_id", - # "observation_date", "location_description", "boundary_description", - "new_occurrence", - "boundary", "mapped_boundary", "buffer_radius", "datum_id", @@ -790,8 +784,6 @@ class Meta: "region_id", "district_id", "locality", - # 'geojson_point', - # 'geojson_polygon', "has_boundary", "has_points", ) @@ -1917,9 +1909,6 @@ class SaveOCRLocationSerializer(serializers.ModelSerializer): datum_id = serializers.IntegerField(required=False, allow_null=True) coordinate_source_id = serializers.IntegerField(required=False, allow_null=True) location_accuracy_id = serializers.IntegerField(required=False, allow_null=True) - # observation_date = serializers.DateTimeField( - # format="%Y-%m-%d %H:%M:%S", required=False, allow_null=True - # ) has_boundary = serializers.SerializerMethodField() has_points = serializers.SerializerMethodField() @@ -1928,11 +1917,8 @@ class Meta: fields = ( "id", "occurrence_report_id", - # "observation_date", "location_description", "boundary_description", - "new_occurrence", - "boundary", "mapped_boundary", "buffer_radius", "datum_id", @@ -1941,7 +1927,6 @@ class Meta: "region_id", "district_id", "locality", - # 'geojson_polygon', "has_boundary", "has_points", ) @@ -3226,7 +3211,6 @@ class Meta: "copied_ocr", "location_description", "boundary_description", - "boundary", "mapped_boundary", "buffer_radius", "datum_id", @@ -3474,7 +3458,6 @@ class Meta: "occurrence_id", "location_description", "boundary_description", - "boundary", "mapped_boundary", "buffer_radius", "datum_id", From ecc7b74761fc571d8bd5b406c4d7ac5f7e95a00a Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 6 Sep 2024 10:12:31 +0800 Subject: [PATCH 11/29] Remove references to unused fields. --- .../common/occurrence/occ_locations.vue | 17 +---------------- .../common/occurrence/ocr_location.vue | 17 +---------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/boranga/frontend/boranga/src/components/common/occurrence/occ_locations.vue b/boranga/frontend/boranga/src/components/common/occurrence/occ_locations.vue index e4e3c40f..afbba120 100644 --- a/boranga/frontend/boranga/src/components/common/occurrence/occ_locations.vue +++ b/boranga/frontend/boranga/src/components/common/occurrence/occ_locations.vue @@ -182,22 +182,7 @@ - {field} is a relationship") + + # If it does, set the relationship + setattr( + current_model_instance, + field.name, + potential_parent_instance, + ) + related_to_parent = True + break + + if related_to_parent: + break + + # If we didn't find a relationship in the current model, search the parent model + for field in potential_parent_instance.__class__._meta.get_fields(): + if field.related_model == current_model_instance: + logger.debug(f" ---> {field} is a relationship") + + # If it does, set the relationship + setattr( + current_model_instance, + field.name, + potential_parent_instance, + ) + related_to_parent = True + break + + if related_to_parent: + break + + if not related_to_parent: + error_message = ( + "Could not find a parent model to relate this model to " + "(Probably due to an error saving the parent model instance)" + ) + errors.append( + { + "row_index": index, + "error_type": "relationship", + "data": model_data, + "error_message": error_message, + } + ) + return + + try: + current_model_instance.save() + model_instances[current_model_instance._meta.model_name] = ( + current_model_instance + ) + logger.debug(f"Model instance created: {current_model_instance}") + except IntegrityError as e: + logger.error(f"Error creating model instance: {e}") + errors.append( + { + "row_index": index, + "error_type": "integrity", + "data": model_data, + "error_message": f"Error creating model instance: {e}", + } + ) + return def retry(self): From 677c121ab6dc11c7bac7c02e39cbcbdc337e2b98 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 10 Sep 2024 14:12:33 +0800 Subject: [PATCH 24/29] Add django_lookup_field_name field to OccurrenceReportBulkImportSchemaColumn model. --- boranga/components/occurrence/models.py | 1 + ...column_django_lookup_field_name_and_more.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 boranga/migrations/0458_occurrencereportbulkimportschemacolumn_django_lookup_field_name_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 4ee6def9..24adce4e 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -5937,6 +5937,7 @@ class OccurrenceReportBulkImportSchemaColumn(OrderedModel): related_name="import_columns", ) django_import_field_name = models.CharField(max_length=50, blank=False, null=False) + django_lookup_field_name = models.CharField(max_length=50, blank=True, null=True) # The name of the column header in the .xlsx file xlsx_column_header_name = models.CharField(max_length=50, blank=False, null=False) diff --git a/boranga/migrations/0458_occurrencereportbulkimportschemacolumn_django_lookup_field_name_and_more.py b/boranga/migrations/0458_occurrencereportbulkimportschemacolumn_django_lookup_field_name_and_more.py new file mode 100644 index 00000000..1322e625 --- /dev/null +++ b/boranga/migrations/0458_occurrencereportbulkimportschemacolumn_django_lookup_field_name_and_more.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.9 on 2024-09-10 06:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0457_remove_occurrence_review_date_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="occurrencereportbulkimportschemacolumn", + name="django_lookup_field_name", + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] From 35be65d170b0df84c848f232c083fd2465f19fc4 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 10 Sep 2024 15:31:49 +0800 Subject: [PATCH 25/29] Make the django_lookup_field_name field default to 'id'. --- boranga/components/occurrence/models.py | 4 +++- ..._alter_communitydocument__file_and_more.py | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 boranga/migrations/0459_alter_communitydocument__file_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 24adce4e..a39ad6ad 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -5937,7 +5937,9 @@ class OccurrenceReportBulkImportSchemaColumn(OrderedModel): related_name="import_columns", ) django_import_field_name = models.CharField(max_length=50, blank=False, null=False) - django_lookup_field_name = models.CharField(max_length=50, blank=True, null=True) + django_lookup_field_name = models.CharField( + max_length=50, default="id", blank=True, null=True + ) # The name of the column header in the .xlsx file xlsx_column_header_name = models.CharField(max_length=50, blank=False, null=False) diff --git a/boranga/migrations/0459_alter_communitydocument__file_and_more.py b/boranga/migrations/0459_alter_communitydocument__file_and_more.py new file mode 100644 index 00000000..ad9b75a7 --- /dev/null +++ b/boranga/migrations/0459_alter_communitydocument__file_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 5.0.9 on 2024-09-10 07:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "boranga", + "0458_occurrencereportbulkimportschemacolumn_django_lookup_field_name_and_more", + ), + ] + + operations = [ + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="django_lookup_field_name", + field=models.CharField(blank=True, default="id", max_length=50, null=True), + ), + ] From 06ceadd40415a4244831a23a1cb904ecddaaf17c Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 10 Sep 2024 15:40:31 +0800 Subject: [PATCH 26/29] Add lookup field options as an attribute for fields that are related. --- boranga/components/main/serializers.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/boranga/components/main/serializers.py b/boranga/components/main/serializers.py index 1e7835de..7f93ded9 100755 --- a/boranga/components/main/serializers.py +++ b/boranga/components/main/serializers.py @@ -170,6 +170,17 @@ def filter_fields(field): xlsx_validation_type = get_openpyxl_data_validation_type_for_django_field( field ) + lookup_field_options = None + if hasattr(field, "related_model") and field.related_model: + related_model = field.related_model + fields = related_model._meta.get_fields() + lookup_field_options = [ + field.verbose_name.lower() + for field in related_model._meta.get_fields() + if not field.related_model + and field.unique + and not field.name.endswith("_number") + ] model_fields.append( { "name": field.name, @@ -179,6 +190,7 @@ def filter_fields(field): "allow_null": allow_null, "max_length": max_length, "xlsx_validation_type": xlsx_validation_type, + "lookup_field_options": lookup_field_options, } ) return model_fields From 8995a092e1ae27701bb56c302ab800befd230cb9 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 10 Sep 2024 15:54:35 +0800 Subject: [PATCH 27/29] Code to allow selection of django lookup field or entry of custom field for advanced uses. Also bug fix that was breaking popovers when the first column was selected. --- .../occurrence/bulk_import_schema.vue | 108 ++++++++++++------ 1 file changed, 70 insertions(+), 38 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue index 1c571bf3..63d9e40d 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue @@ -151,7 +151,7 @@ -
@@ -167,7 +167,7 @@ Add a Single Field
-
+
@@ -280,7 +280,7 @@ class=""> {{ choice[0] - }} + }} @@ -328,36 +328,40 @@
-
+
+ :data-bs-content="`Which field to use when looking up the ${selectedField.display_name} record.
(Advanced: You can use a custom field name if it's not in the list and it may span model relationships e.g. 'taxonomy__scientific_name' )`" + data-bs-placement="top" :data-bs-html="true">
-
- - Type - -
-
- - Lookup Model - -
Lookup Field - + + + +
+
@@ -441,9 +445,11 @@ export default { selectedColumnIndex: null, selectedContentType: null, selectedField: null, + customLookupField: false, + previousCustomLookupField: null, sortable: null, addEditMode: false, - addingSingleColumn: false, + showDjangoImportFieldSelect: false, newColumn: null, saving: false, errors: null @@ -453,7 +459,7 @@ export default { alert }, computed: { - showValidationFields() { + showDjangoLookupField() { return this.selectedColumn && this.selectedColumn.xlsx_column_header_name && this.selectedColumn.django_import_content_type && @@ -536,7 +542,7 @@ export default { if (!this.selectedColumn.django_import_content_type) { this.selectedField = null this.selectedContentType = null - this.addingSingleColumn = false + this.showDjangoImportFieldSelect = false this.$refs['django-import-field'].focus() return } @@ -569,12 +575,29 @@ export default { this.$refs['column-name'].focus() }) }, + selectLookupField() { + if (this.selectedColumn.django_lookup_field_name == 'custom') { + this.customLookupField = true + if (this.previousCustomLookupField) { + this.selectedColumn.django_lookup_field_name = this.previousCustomLookupField + } else { + this.selectedColumn.django_lookup_field_name = '' + } + this.$nextTick(() => { + this.$refs['custom-lookup-field'].focus() + }) + } else { + this.customLookupField = false + this.selectedColumn.django_lookup_field_name = 'id' + } + }, getNewColumnData() { return { id: null, schema: this.schema.id, django_import_content_type: '', django_import_field_name: '', + django_lookup_field_name: null, xlsx_column_header_name: '', xlsx_data_validation_allow_blank: true, default_value: null, @@ -583,7 +606,7 @@ export default { } }, addSingleColumn() { - this.addingSingleColumn = true + this.showDjangoImportFieldSelect = true this.$nextTick(() => { this.enablePopovers(); this.$refs['django-import-field'].focus() @@ -652,21 +675,30 @@ export default { }, selectColumn(column) { + if (column.django_import_content_type) { + this.selectedContentType = this.djangoContentTypes.filter( + djangoContentType => djangoContentType.id == column.django_import_content_type + )[0] + if (column.django_import_field_name) { + this.selectedField = this.selectedContentType.model_fields.filter( + modelField => modelField.name == column.django_import_field_name + )[0] + if (this.selectedField.lookup_field_options && !this.selectedField.lookup_field_options.includes(column.django_lookup_field_name)) { + this.customLookupField = true + if (column.django_lookup_field_name) { + this.previousCustomLookupField = column.django_lookup_field_name + } + } + } + } this.selectedColumn = column this.selectedColumnIndex = this.schema.columns.indexOf(column) this.addEditMode = true + if (this.selectedColumn.id) { + this.showDjangoImportFieldSelect = true + } this.$nextTick(() => { this.enablePopovers(); - if (this.selectedColumn.django_import_content_type) { - this.selectedContentType = this.djangoContentTypes.filter( - djangoContentType => djangoContentType.id == this.selectedColumn.django_import_content_type - )[0] - if (this.selectedColumn.django_import_field_name) { - this.selectedField = this.selectedContentType.model_fields.filter( - modelField => modelField.name == this.selectedColumn.django_import_field_name - )[0] - } - } this.$refs['django-import-model'].focus() }) }, @@ -675,7 +707,7 @@ export default { this.selectedColumn = null this.selectedColumnIndex = null this.addEditMode = false - this.addingSingleColumn = false + this.showDjangoImportFieldSelect = false }, removeColumn(column) { let columnTitle = column.xlsx_column_header_name ? `Are you sure you want to delete column: ${column.xlsx_column_header_name}?` : `Are you sure you want to delete this column?` From ab33b5a1888ae35811518bfd5e519bcec9b296d5 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 11:08:44 +0800 Subject: [PATCH 28/29] Add function that returns the display field for a model. --- boranga/helpers.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/boranga/helpers.py b/boranga/helpers.py index 439e4da0..4a804fb1 100755 --- a/boranga/helpers.py +++ b/boranga/helpers.py @@ -489,3 +489,29 @@ def convert_internal_url_to_external_url(url): # remove '-internal'. This email is for external submitters url = "".join(url.split(settings.SITE_SUBDOMAIN_INTERNAL_SUFFIX)) return url + + +def get_display_field_for_model(model: models.Model) -> str: + """ + Returns the field name to display for a model in the admin list display. + """ + # Find the best field to use for a display value + display_field = None + fields = model._meta.get_fields() + for field in fields: + if field.name in settings.OCR_BULK_IMPORT_LOOKUP_TABLE_DISPLAY_FIELDS: + display_field = field.name + break + + if not display_field: + # If we can't find a display field, we'll just use the first CharField we find + for field in fields: + if isinstance(field, models.fields.CharField): + display_field = field.name + break + + if not display_field: + # Fall back to the id + display_field = "id" + + return display_field From aa45c356fd85e9042a69bbe3d633d817d05f2dd8 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 11:11:25 +0800 Subject: [PATCH 29/29] Column validation now also performs the foreign key lookup when necessary and replaces the cell value with an instance of the associated model. --- boranga/components/occurrence/models.py | 90 ++++++++++++++++++------- 1 file changed, 67 insertions(+), 23 deletions(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index a39ad6ad..eebafd92 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -78,6 +78,7 @@ ) from boranga.helpers import ( clone_model, + get_display_field_for_model, is_occurrence_approver, is_occurrence_assessor, member_ids, @@ -5477,7 +5478,9 @@ def process_row(self, index, row, errors): cell_value = row[index] - column_error_count += column.validate(cell_value, index, errors) + cell_value, errors_added = column.validate(cell_value, index, errors) + + column_error_count += errors_added row_error_count += column_error_count total_column_error_count += column_error_count @@ -5821,27 +5824,7 @@ def preview_import_file(self): # Instead, the field will be validated during the import process continue - # Find the best field to use for a display value - display_field = None - fields = related_model._meta.get_fields() - for field in fields: - if ( - field.name - in settings.OCR_BULK_IMPORT_LOOKUP_TABLE_DISPLAY_FIELDS - ): - display_field = field.name - break - - if not display_field: - # If we can't find a display field, we'll just use the first CharField we find - for field in fields: - if isinstance(field, models.fields.CharField): - display_field = field.name - break - - if not display_field: - # Fall back to the id - display_field = "id" + display_field = get_display_field_for_model(related_model) dv = DataValidation( type=dv_types["list"], @@ -6105,7 +6088,68 @@ def validate(self, cell_value, index, errors): ) errors_added += 1 - return errors_added + model_class = apps.get_model("boranga", self.django_import_content_type.model) + if hasattr(model_class, self.django_import_field_name): + field = model_class._meta.get_field(self.django_import_field_name) + if isinstance(field, models.ForeignKey): + related_model = field.related_model + related_model_qs = related_model.objects.all() + + # Check if the related model is Archivable + if issubclass(related_model, ArchivableModel): + related_model_qs = related_model_qs.exclude(archived=True) + + if not related_model_qs.exists() or related_model_qs.count() == 0: + return cell_value, errors_added + + if ( + related_model_qs.count() + > settings.OCR_BULK_IMPORT_LOOKUP_TABLE_RECORD_LIMIT + ): + # Use the django lookup field to find the value + lookup_field = self.django_lookup_field_name + try: + related_model_instance = related_model_qs.get( + **{lookup_field: cell_value} + ) + except related_model.DoesNotExist: + error_message = ( + f"Can't find {self.django_import_field_name} record by looking up " + f"{self.django_lookup_field_name} with value {cell_value} " + f"for column {self.column_header_name}" + ) + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": error_message, + } + ) + errors_added += 1 + return cell_value, errors_added + + # Replace the lookup cell_value with the actual instance to assigned + cell_value = related_model_instance + return cell_value, errors_added + + display_field = get_display_field_for_model(related_model) + + if cell_value not in related_model_qs.values_list( + display_field, flat=True + ): + error_message = f"Value {cell_value} in column {self.column_header_name} is not in the lookup table" + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": error_message, + } + ) + errors_added += 1 + + return cell_value, errors_added # Occurrence Report Document