From bf8d92b3f81a3749a9920bf941942a252f804684 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 16:57:45 +0800 Subject: [PATCH 001/185] npm audit fix. --- boranga/frontend/boranga/package-lock.json | 122 ++++++++++++++++----- 1 file changed, 94 insertions(+), 28 deletions(-) diff --git a/boranga/frontend/boranga/package-lock.json b/boranga/frontend/boranga/package-lock.json index a977fd54..2ece9db9 100644 --- a/boranga/frontend/boranga/package-lock.json +++ b/boranga/frontend/boranga/package-lock.json @@ -3757,9 +3757,9 @@ "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", @@ -3769,7 +3769,7 @@ "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.11.0", + "qs": "6.13.0", "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" @@ -3800,6 +3800,20 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/body-parser/node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/bonjour-service": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.2.1.tgz", @@ -5832,36 +5846,36 @@ } }, "node_modules/express": { - "version": "4.19.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", - "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", + "version": "4.20.0", + "resolved": "https://registry.npmjs.org/express/-/express-4.20.0.tgz", + "integrity": "sha512-pLdae7I6QqShF5PnNTCVn4hI91Dx0Grkn2+IAsMTgMIKuQVte2dN9PeGSSAME2FR8anOhVA62QDIUaWVfEXVLw==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.2", + "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "1.2.0", "fresh": "0.5.2", "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", + "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", + "path-to-regexp": "0.1.10", "proxy-addr": "~2.0.7", "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", + "send": "0.19.0", + "serve-static": "1.16.0", "setprototypeof": "1.2.0", "statuses": "2.0.1", "type-is": "~1.6.18", @@ -5880,6 +5894,14 @@ "ms": "2.0.0" } }, + "node_modules/express/node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/express/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -7650,9 +7672,12 @@ } }, "node_modules/merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/merge-source-map": { "version": "1.1.0", @@ -8507,9 +8532,9 @@ "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, "node_modules/path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==" }, "node_modules/path-type": { "version": "4.0.0", @@ -9835,9 +9860,9 @@ "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==" }, "node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -9954,9 +9979,9 @@ } }, "node_modules/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.0.tgz", + "integrity": "sha512-pDLK8zwl2eKaYrs8mrPZBJua4hMplRWJ1tIFksVC3FtBEBnl8dxgeHtsaMS8DhS9i4fLObaon6ABoc4/hQGdPA==", "dependencies": { "encodeurl": "~1.0.2", "escape-html": "~1.0.3", @@ -9967,6 +9992,47 @@ "node": ">= 0.8.0" } }, + "node_modules/serve-static/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/serve-static/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/serve-static/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/serve-static/node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -10372,9 +10438,9 @@ } }, "node_modules/sweetalert2": { - "version": "11.13.2", - "resolved": "https://registry.npmjs.org/sweetalert2/-/sweetalert2-11.13.2.tgz", - "integrity": "sha512-Q361QVJrDce3pA+46m2JmfDQyxxlmVE6t7ScoMwubm2PQKTlUqaMpzWq/DZRSPL8Sg2hUCzUAXQ9dwMPnbsy7Q==", + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/sweetalert2/-/sweetalert2-11.14.0.tgz", + "integrity": "sha512-kF1Q/+GtZZXr+rYVcBNwlEsnxP089CpDbck+MYjvLaQj9x4fzHqN9UhlkHOIR0k09LVu2sx8cU9BnvRlxWIZqg==", "funding": { "type": "individual", "url": "https://github.com/sponsors/limonte" From 8ca900e58ac7aa91045ccfbb2743f401a21f04d9 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 16:58:26 +0800 Subject: [PATCH 002/185] Add MultiSelectField and ForeignKey handling to get_openpyxl_data_validation_type_for_django_field. --- boranga/helpers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/boranga/helpers.py b/boranga/helpers.py index 4a804fb1..6150d65d 100755 --- a/boranga/helpers.py +++ b/boranga/helpers.py @@ -11,6 +11,7 @@ from django.db.models import Q from ledger_api_client.ledger_models import EmailUserRO as EmailUser from ledger_api_client.managed_models import SystemGroup +from multiselectfield import MultiSelectField from boranga.settings import ( DJANGO_ADMIN_GROUP, @@ -411,11 +412,14 @@ def get_openpyxl_data_validation_type_for_django_field(field): models.IntegerField: "whole", models.DecimalField: "decimal", models.BooleanField: "list", + models.ForeignKey: "list", models.DateField: "date", models.DateTimeField: "date", } - if isinstance(field, models.CharField) and field.choices: + if isinstance(field, MultiSelectField) or ( + isinstance(field, models.CharField) and field.choices + ): return dv_types["list"] for django_field, dv_type in field_type_map.items(): From 3d1f845fca1c6836bd6bd072a5841f4f7bf2bbf1 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 17:04:14 +0800 Subject: [PATCH 003/185] Add foreign_key_count and requires_lookup_field to OccurrenceReportBulkImportSchemaColumnNestedSerializer. --- boranga/components/occurrence/serializers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index 23aa866a..4b3277f2 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -3844,7 +3844,6 @@ def get_last_updated_by(self, obj): class OccurrenceReportBulkImportSchemaColumnSerializer(serializers.ModelSerializer): - class Meta: model = OccurrenceReportBulkImportSchemaColumn fields = "__all__" @@ -3856,6 +3855,8 @@ class OccurrenceReportBulkImportSchemaColumnNestedSerializer( ): id = serializers.IntegerField(allow_null=True, required=False) order = serializers.IntegerField() + foreign_key_count = serializers.IntegerField(read_only=True) + requires_lookup_field = serializers.BooleanField(read_only=True) class Meta: model = OccurrenceReportBulkImportSchemaColumn From 1437248cc7688ca542a60865cbaeb86e58e4b28b Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 17:05:43 +0800 Subject: [PATCH 004/185] Add handling for MultiSelectField and ForeignKey to the get_model_fields method on ContentTypeSerializer. --- boranga/components/main/serializers.py | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/boranga/components/main/serializers.py b/boranga/components/main/serializers.py index 7f93ded9..7e875b5a 100755 --- a/boranga/components/main/serializers.py +++ b/boranga/components/main/serializers.py @@ -1,17 +1,21 @@ import logging +from django.conf import settings from django.contrib.contenttypes.models import ContentType +from django.db import models from django.db.models.fields.related import ForeignKey, OneToOneField from ledger_api_client.ledger_models import EmailUserRO from ledger_api_client.ledger_models import EmailUserRO as EmailUser from rest_framework import serializers from boranga.components.main.models import ( + ArchivableModel, CommunicationsLogEntry, GlobalSettings, HelpTextEntry, ) from boranga.helpers import ( + get_display_field_for_model, get_openpyxl_data_validation_type_for_django_field, is_django_admin, ) @@ -165,13 +169,20 @@ def filter_fields(field): ) field_type = str(type(field)).split(".")[-1].replace("'>", "") choices = field.choices if hasattr(field, "choices") else None + if field_type == "MultiSelectField": + # Have to create an instance for the choices to be populated :-( + # as for some reason they are populated in the __init__ method + instance = obj.model_class()() + multi_select_field = instance._meta.get_field(field.name) + choices = multi_select_field.choices + allow_null = field.null if hasattr(field, "null") else None max_length = field.max_length if hasattr(field, "max_length") else None xlsx_validation_type = get_openpyxl_data_validation_type_for_django_field( field ) lookup_field_options = None - if hasattr(field, "related_model") and field.related_model: + if isinstance(field, models.ForeignKey): related_model = field.related_model fields = related_model._meta.get_fields() lookup_field_options = [ @@ -181,6 +192,24 @@ def filter_fields(field): and field.unique and not field.name.endswith("_number") ] + + related_model_qs = related_model.objects.all() + + if issubclass(related_model, ArchivableModel): + related_model_qs = related_model_qs.filter(archived=False) + + related_model_count = related_model_qs.count() + + if ( + related_model_count == 0 + or related_model_count + > settings.OCR_BULK_IMPORT_LOOKUP_TABLE_RECORD_LIMIT + ): + choices = None + else: + display_field = get_display_field_for_model(related_model) + choices = list(related_model_qs.values_list("id", display_field)) + model_fields.append( { "name": field.name, From 524d15a006d256fa133257c006fd902e1610a079 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 17:06:26 +0800 Subject: [PATCH 005/185] Add support for MultiSelectField to the preview_import_file method on OccurrenceReportBulkImportSchema. --- boranga/components/occurrence/models.py | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index eebafd92..cb1417b8 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -2287,6 +2287,8 @@ class OCRHabitatComposition(models.Model): null=True, related_name="habitat_composition", ) + # TODO: Consider fixing these to use a function that returns the choices + # as setting them in the __init__ method creates issues in other parts of the application land_form = MultiSelectField(max_length=250, blank=True, choices=[], null=True) rock_type = models.ForeignKey( RockType, on_delete=models.SET_NULL, null=True, blank=True @@ -5731,6 +5733,25 @@ def preview_import_file(self): prompt="Either leave the field blank or enter the default value", promptTitle="Value", ) + elif isinstance( + model_field, MultiSelectField + ): # MultiSelectField is a custom field, not a standard Django field + + # Have to create an instance for the choices to be populated :-( + # as for some reason they are populated in the __init__ method + instance = column.django_import_content_type.model_class()() + multi_select_field = instance._meta.get_field(model_field.name) + choices = multi_select_field.choices + + dv = DataValidation( + type=dv_types["list"], + allow_blank=model_field.null, + formula1=",".join([choice[1] for choice in choices]), + error="Please select a valid option from the list", + errorTitle="Invalid selection", + prompt="Select a value from the list", + promptTitle="List selection", + ) elif ( isinstance(model_field, models.fields.CharField) and model_field.choices ): @@ -5996,6 +6017,33 @@ class Meta(OrderedModel.Meta): def __str__(self): return f"{self.xlsx_column_header_name} - {self.schema}" + @property + def foreign_key_count(self): + if not self.django_import_content_type or not self.django_import_field_name: + return 0 + + field = self.django_import_content_type.model_class()._meta.get_field( + self.django_import_field_name + ) + if not isinstance(field, models.ForeignKey): + return 0 + + related_model_qs = field.related_model.objects.all() + + if issubclass(field.related_model, ArchivableModel): + related_model_qs = field.related_model.objects.exclude(archived=True) + + return related_model_qs.count() + + @property + def requires_lookup_field(self): + if not self.django_import_content_type or not self.django_import_field_name: + return False + + return ( + self.foreign_key_count > settings.OCR_BULK_IMPORT_LOOKUP_TABLE_RECORD_LIMIT + ) + def validate(self, cell_value, index, errors): errors_added = 0 if not self.xlsx_data_validation_allow_blank and not cell_value: From 84e26f6c2bc30e7351ed5e524b56c10c28c8cb31 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 11 Sep 2024 17:11:59 +0800 Subject: [PATCH 006/185] Move the preview list options modal higher up the dom tree so the backdrop renders properly. Make preview list options render properly for MultiSelectField and ForeignKey. Fix a few bugs/regressions. --- .../occurrence/bulk_import_schema.vue | 83 ++++++++++--------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue index 63d9e40d..be0d2806 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue @@ -256,44 +256,6 @@ data-bs-target="#preview-choices"> Preview List Choices - @@ -423,6 +385,45 @@ + + @@ -463,6 +464,7 @@ export default { return this.selectedColumn && this.selectedColumn.xlsx_column_header_name && this.selectedColumn.django_import_content_type && + this.selectedColumn.requires_lookup_field && this.selectedColumn.django_import_field_name && this.selectedField && this.selectedField.type === 'ForeignKey'; }, }, @@ -588,7 +590,7 @@ export default { }) } else { this.customLookupField = false - this.selectedColumn.django_lookup_field_name = 'id' + this.selectedColumn.django_lookup_field_name = this.$refs['lookup-field'].value } }, getNewColumnData() { @@ -620,6 +622,7 @@ export default { this.selectedColumn = this.newColumn this.selectedColumnIndex = this.schema.columns.indexOf(this.newColumn) this.addEditMode = true + this.showDjangoImportFieldSelect = false this.$nextTick(() => { this.enablePopovers(); this.$refs['django-import-model'].focus() @@ -694,7 +697,7 @@ export default { this.selectedColumn = column this.selectedColumnIndex = this.schema.columns.indexOf(column) this.addEditMode = true - if (this.selectedColumn.id) { + if (this.selectedColumn.id && column.django_import_field_name) { this.showDjangoImportFieldSelect = true } this.$nextTick(() => { From 205cf6b3ee3ab73ca98b4a51114f4d7e55657367 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 12 Sep 2024 12:49:31 +0800 Subject: [PATCH 007/185] Register route for OccurrenceReportBulkImportSchemaColumnViewSet. --- boranga/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/boranga/urls.py b/boranga/urls.py index 816d2e38..7576be41 100755 --- a/boranga/urls.py +++ b/boranga/urls.py @@ -160,6 +160,11 @@ def trigger_error(request): occurrence_api.OccurrenceReportBulkImportSchemaViewSet, "occurrence_report_bulk_import_schemas", ) +router.register( + r"occurrence_report_bulk_import_schema_columns", + occurrence_api.OccurrenceReportBulkImportSchemaColumnViewSet, + "occurrence_report_bulk_import_schema_columns", +) router.register(r"observer_detail", occurrence_api.ObserverDetailViewSet) router.register(r"contact_detail", occurrence_api.ContactDetailViewSet) router.register(r"occurrence_sites", occurrence_api.OccurrenceSiteViewSet) From 77aaec7c65916686803f6d0152a2f2971e4cd362 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 12 Sep 2024 12:50:22 +0800 Subject: [PATCH 008/185] Add OccurrenceReportBulkImportSchemaColumnViewSet with preview_foreign_key_values_xlsx detail route. --- boranga/components/occurrence/api.py | 68 ++++++++++++---------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/boranga/components/occurrence/api.py b/boranga/components/occurrence/api.py index bbda71b5..7873f47e 100644 --- a/boranga/components/occurrence/api.py +++ b/boranga/components/occurrence/api.py @@ -6390,21 +6390,6 @@ def copy(self, request, *args, **kwargs): serializer = OccurrenceReportBulkImportSchemaSerializer(new_instance) return Response(serializer.data, status=status.HTTP_201_CREATED) - @detail_route(methods=["put"], detail=True) - def save_column(self, request, *args, **kwargs): - instance = self.get_object() - column_data = request.data.get("column_data", None) - if not column_data: - raise serializers.ValidationError("Column data is required") - serializer = OccurrenceReportBulkImportSchemaColumnSerializer( - instance, data=column_data - ) - serializer.is_valid(raise_exception=True) - serializer.save() - - serializer = OccurrenceReportBulkImportSchemaSerializer(instance) - return Response(serializer.data, status=status.HTTP_201_CREATED) - @detail_route(methods=["get"], detail=False) def default_value_choices(self, request, *args, **kwargs): default_value_field = OccurrenceReportBulkImportSchemaColumn._meta.get_field( @@ -6412,31 +6397,36 @@ def default_value_choices(self, request, *args, **kwargs): ) return Response(default_value_field.choices, status=status.HTTP_200_OK) - @detail_route(methods=["patch"], detail=True) - def reorder_column(self, request, *args, **kwargs): - instance = self.get_object() - # Don't order columns that haven't been saved - pk = request.data.get("id", None) - if not pk: - raise serializers.ValidationError( - "column must have id field to be reordered (i.e. record must be saved first)" - ) +class OccurrenceReportBulkImportSchemaColumnViewSet( + viewsets.GenericViewSet, + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.UpdateModelMixin, + mixins.ListModelMixin, +): + queryset = OccurrenceReportBulkImportSchemaColumn.objects.all() + serializer_class = OccurrenceReportBulkImportSchemaColumnSerializer + permission_classes = [OccurrenceReportBulkImportPermission] - order = request.data.get("order", None) - if order is None: - raise serializers.ValidationError("order field is missing from column") + @detail_route(methods=["get"], detail=True) + def preview_foreign_key_values_xlsx(self, request, *args, **kwargs): + instance = self.get_object() - try: - column = instance.columns.get(pk=pk) - except OccurrenceReportBulkImportSchemaColumn.DoesNotExist: - raise serializers.ValidationError( - f"Column with id {pk} not found in schema" + buffer = BytesIO() + workbook = instance.preview_foreign_key_values_xlsx + if not workbook: + return Response( + {"message": "No foreign key values to preview"}, + status=status.HTTP_404_NOT_FOUND, ) - - column.to(order) - - # instance.refresh_from_db() - - serializer = OccurrenceReportBulkImportSchemaSerializer(instance) - return Response(serializer.data, status=status.HTTP_201_CREATED) + workbook.save(buffer) + buffer.seek(0) + filename = ( + f"{instance.django_import_content_type.model}-{instance.django_import_field_name}" + f"-foreign-key-list-values.xlsx" + ) + response = HttpResponse(buffer.read(), content_type="application/vnd.ms-excel") + response["Content-Disposition"] = f"attachment; filename={filename}" + buffer.close() + return response From df07b9d454270591040fa1e4970007e87acb14d9 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 12 Sep 2024 12:52:33 +0800 Subject: [PATCH 009/185] Add preview_foreign_key_values_xlsx method to OccurrenceReportBulkImportSchemaColumn. --- boranga/components/occurrence/models.py | 52 ++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index cb1417b8..f20e21fc 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -24,7 +24,7 @@ from django.core.validators import MaxValueValidator, MinValueValidator from django.db import IntegrityError, models, transaction from django.db.models import CharField, Count, Func, Max, Q -from django.db.models.functions import Cast +from django.db.models.functions import Cast, Length from django.utils import timezone from ledger_api_client.ledger_models import EmailUserRO as EmailUser from ledger_api_client.managed_models import SystemGroup @@ -6044,6 +6044,56 @@ def requires_lookup_field(self): self.foreign_key_count > settings.OCR_BULK_IMPORT_LOOKUP_TABLE_RECORD_LIMIT ) + @property + def preview_foreign_key_values_xlsx(self): + if not self.django_import_content_type or not self.django_import_field_name: + return None + + field = self.django_import_content_type.model_class()._meta.get_field( + self.django_import_field_name + ) + if not isinstance(field, models.ForeignKey): + return None + + related_model = field.related_model + + if self.django_lookup_field_name: + display_field = self.django_lookup_field_name + else: + display_field = get_display_field_for_model(related_model) + + filter_dict = {f"{display_field}__isnull": False} + related_model_qs = related_model.objects.filter(**filter_dict) + + if issubclass(related_model, ArchivableModel): + related_model_qs = related_model.objects.exclude(archived=True) + + workbook = openpyxl.Workbook() + worksheet = workbook.active + + # Query the max characer length of the display field + max_length = related_model_qs.aggregate( + max_length=Max(Length(Cast(display_field, output_field=CharField()))) + )["max_length"] + + if len(self.xlsx_column_header_name) > max_length: + max_length = len(self.xlsx_column_header_name) + + headers = [self.xlsx_column_header_name] + worksheet.append(headers) + for cell_value in related_model_qs.order_by(display_field).values_list( + display_field, flat=True + ): + worksheet.append([cell_value]) + + # Make the headers bold + worksheet["A1"].font = Font(bold=True) + + # Make the column widths appropriate + worksheet.column_dimensions["A"].width = max_length + 2 + + return workbook + def validate(self, cell_value, index, errors): errors_added = 0 if not self.xlsx_data_validation_allow_blank and not cell_value: From 0310c3b4540c417cf5fd3aa13ac2fe191ee24718 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 12 Sep 2024 12:54:01 +0800 Subject: [PATCH 010/185] Incorporate new preview_foreign_key_values_xlsx end point for foreign key columns. --- .../occurrence/bulk_import_schema.vue | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue index be0d2806..9c29363a 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue @@ -250,9 +250,9 @@ DD/MM/YYYY HH:MM:SS - From 987d09b65733877f8fee21cc41adee371ec85b75 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 12 Sep 2024 16:51:41 +0800 Subject: [PATCH 011/185] Assign request user as submitter of bulk import task by default. Modify column validate method to get validation information from the django field directly rather than from attributes stored on the model. Add handling for MultiSelectField to column validate. --- boranga/components/occurrence/models.py | 202 ++++++++++++++++-------- 1 file changed, 133 insertions(+), 69 deletions(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index f20e21fc..9976de64 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -79,6 +79,7 @@ from boranga.helpers import ( clone_model, get_display_field_for_model, + get_openpyxl_data_validation_type_for_django_field, is_occurrence_approver, is_occurrence_assessor, member_ids, @@ -5538,6 +5539,7 @@ def process_row(self, index, row, errors): current_model_instance.bulk_import_task_id = self.pk current_model_instance.import_hash = row_hash current_model_instance.group_type_id = self.schema.group_type_id + current_model_instance.submitter = self.email_user else: current_model_instance.pk = OccurrenceReport.objects.get( migrated_from_id=row[0] @@ -5736,22 +5738,10 @@ def preview_import_file(self): elif isinstance( model_field, MultiSelectField ): # MultiSelectField is a custom field, not a standard Django field - - # Have to create an instance for the choices to be populated :-( - # as for some reason they are populated in the __init__ method - instance = column.django_import_content_type.model_class()() - multi_select_field = instance._meta.get_field(model_field.name) - choices = multi_select_field.choices - - dv = DataValidation( - type=dv_types["list"], - allow_blank=model_field.null, - formula1=",".join([choice[1] for choice in choices]), - error="Please select a valid option from the list", - errorTitle="Invalid selection", - prompt="Select a value from the list", - promptTitle="List selection", - ) + # Unfortunately there is no easy way to embed validation in .xlsx + # for a comma separated list of values so this will be validated + # during the import process + continue elif ( isinstance(model_field, models.fields.CharField) and model_field.choices ): @@ -6096,6 +6086,37 @@ def preview_foreign_key_values_xlsx(self): def validate(self, cell_value, index, errors): errors_added = 0 + + model_class = apps.get_model("boranga", self.django_import_content_type.model) + + if not model_class: + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": f"Model class {self.django_import_content_type.model} not found", + } + ) + errors_added += 1 + return cell_value, errors_added + + if not hasattr(model_class, self.django_import_field_name): + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": f"Field {self.django_import_field_name} not found in model {model_class}", + } + ) + errors_added += 1 + return cell_value, errors_added + + field = model_class._meta.get_field(self.django_import_field_name) + + logger.debug(f"field: {field}") + if not self.xlsx_data_validation_allow_blank and not cell_value: errors.append( { @@ -6107,8 +6128,67 @@ def validate(self, cell_value, index, errors): ) errors_added += 1 - if self.xlsx_data_validation_type == "textLength": - if len(cell_value) > int(self.xlsx_data_validation_formula1): + xlsx_data_validation_type = get_openpyxl_data_validation_type_for_django_field( + field + ) + + if isinstance(field, MultiSelectField): + if not cell_value: + return cell_value, errors_added + + if not isinstance(cell_value, str): + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": f"Value {cell_value} in column {self.column_header_name} is not a string", + } + ) + errors_added += 1 + return cell_value, errors_added + + # Unfortunatly have to have an actual model instance to get the choices + # as they are defined in __init__ + model_instance = model_class() + choices = model_instance._meta.get_field( + self.django_import_field_name + ).choices + + display_values = cell_value.split(",") + logger.debug(f"display_values: {display_values}") + cell_value = [] + for display_value in [ + display_value.strip() for display_value in display_values + ]: + logger.debug(f"display_value: '{display_value}'") + logger.debug([choice[1] for choice in choices]) + if display_value not in [choice[1] for choice in choices]: + errors_message = ( + f"Value '{display_value}' in column {self.xlsx_column_header_name} " + "is not in the list" + ) + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": errors_message, + } + ) + errors_added += 1 + else: + cell_value.append( + [ + choice[0] + for choice in field.choices + if choice[1] == display_value + ][0] + ) + return cell_value, errors_added + + if xlsx_data_validation_type == "textLength" and field.max_length: + if len(str(cell_value)) > field.max_length: error_message = f"Value {cell_value} in column {self.xlsx_column_header_name} has too many characters" errors.append( { @@ -6120,7 +6200,7 @@ def validate(self, cell_value, index, errors): ) errors_added += 1 - if self.xlsx_data_validation_type == "whole": + if xlsx_data_validation_type == "whole": if not isinstance(cell_value, int): errors.append( { @@ -6132,7 +6212,7 @@ def validate(self, cell_value, index, errors): ) errors_added += 1 - if self.xlsx_data_validation_type == "decimal": + if xlsx_data_validation_type == "decimal": try: cell_value = Decimal(cell_value) except Exception: @@ -6146,7 +6226,7 @@ def validate(self, cell_value, index, errors): ) errors_added += 1 - if self.xlsx_data_validation_type == "date": + if xlsx_data_validation_type == "date": try: cell_value = dateutil.parser.parse(cell_value) except Exception: @@ -6160,7 +6240,7 @@ def validate(self, cell_value, index, errors): ) errors_added += 1 - if self.xlsx_data_validation_type == "time": + if xlsx_data_validation_type == "time": try: cell_value = dateutil.parser.parse(cell_value) except Exception: @@ -6174,18 +6254,6 @@ def validate(self, cell_value, index, errors): ) errors_added += 1 - if self.xlsx_data_validation_type == "list": - if cell_value not in self.xlsx_data_validation_formula1: - errors.append( - { - "row_index": index, - "error_type": "column", - "data": cell_value, - "error_message": f"Value {cell_value} in column {self.column_header_name} is not in the list", - } - ) - errors_added += 1 - model_class = apps.get_model("boranga", self.django_import_content_type.model) if hasattr(model_class, self.django_import_field_name): field = model_class._meta.get_field(self.django_import_field_name) @@ -6200,43 +6268,22 @@ def validate(self, cell_value, index, errors): if not related_model_qs.exists() or related_model_qs.count() == 0: return cell_value, errors_added - if ( - related_model_qs.count() - > settings.OCR_BULK_IMPORT_LOOKUP_TABLE_RECORD_LIMIT - ): - # Use the django lookup field to find the value + # Use the django lookup field to find the value + if self.django_lookup_field_name: lookup_field = self.django_lookup_field_name - try: - related_model_instance = related_model_qs.get( - **{lookup_field: cell_value} - ) - except related_model.DoesNotExist: - error_message = ( - f"Can't find {self.django_import_field_name} record by looking up " - f"{self.django_lookup_field_name} with value {cell_value} " - f"for column {self.column_header_name}" - ) - errors.append( - { - "row_index": index, - "error_type": "column", - "data": cell_value, - "error_message": error_message, - } - ) - errors_added += 1 - return cell_value, errors_added - - # Replace the lookup cell_value with the actual instance to assigned - cell_value = related_model_instance - return cell_value, errors_added - - display_field = get_display_field_for_model(related_model) + else: + lookup_field = get_display_field_for_model(related_model) - if cell_value not in related_model_qs.values_list( - display_field, flat=True - ): - error_message = f"Value {cell_value} in column {self.column_header_name} is not in the lookup table" + try: + related_model_instance = related_model_qs.get( + **{lookup_field: cell_value} + ) + except related_model.DoesNotExist: + error_message = ( + f"Can't find {self.django_import_field_name} record by looking up " + f"{self.django_lookup_field_name} with value {cell_value} " + f"for column {self.column_header_name}" + ) errors.append( { "row_index": index, @@ -6246,6 +6293,23 @@ def validate(self, cell_value, index, errors): } ) errors_added += 1 + return cell_value, errors_added + + # Replace the lookup cell_value with the actual instance to assigned + cell_value = related_model_instance + return cell_value, errors_added + + if xlsx_data_validation_type == "list": + if cell_value not in self.xlsx_data_validation_formula1: + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": f"Value {cell_value} in column {self.column_header_name} is not in the list", + } + ) + errors_added += 1 return cell_value, errors_added From 6c7ee84ecd99f83447bf551e15a272b01aeb7fec Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 13 Sep 2024 15:55:39 +0800 Subject: [PATCH 012/185] Add get_geometry_array_from_geojson function. --- boranga/components/spatial/utils.py | 68 ++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/boranga/components/spatial/utils.py b/boranga/components/spatial/utils.py index 77a6ceb4..1605b614 100644 --- a/boranga/components/spatial/utils.py +++ b/boranga/components/spatial/utils.py @@ -13,7 +13,7 @@ import shapely.geometry as shp from django.apps import apps from django.contrib.contenttypes import models as ct_models -from django.contrib.gis.geos import GEOSGeometry +from django.contrib.gis.geos import GEOSGeometry, Polygon from django.core.cache import cache from django.core.exceptions import ValidationError from django.db import IntegrityError @@ -28,6 +28,7 @@ from boranga import settings from boranga.components.occurrence.models import ( BufferGeometry, + GeometryBase, OccurrenceGeometry, OccurrenceTenure, ) @@ -1001,3 +1002,68 @@ def process_proxy(request, remoteurl, queryString, auth_user, auth_password): "Access Denied", content_type="text/html", status=401 ) return http_response + + +def get_geometry_array_from_geojson( + geojson: dict, + cell_value: any, + index: int, + column_name: str, + errors: list, + errors_added: int, +) -> list: + """ + Extracts the geometry array from a GeoJSON object. + """ + if not geojson: + return None + + features = geojson.get("features") + + geoms = [] + bbox = Polygon.from_bbox(GeometryBase.EXTENT) + + for feature in features: + geom = feature.get("geometry") + + if not geom: + error_message = ( + f"Geometry not defined in {cell_value} for column {column_name}" + ) + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": error_message, + } + ) + errors_added += 1 + continue + + logger.debug(f"---> Geometry: {geom}") + logger.debug(f"---> type: {type(geom)}") + + geom = GEOSGeometry(json.dumps(geom)) + + if not geom.within(bbox): + error_message = ( + f"Geomtry defined in {cell_value} for column {column_name} " + "is not within Western Australia" + ) + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": error_message, + } + ) + errors_added += 1 + continue + + geoms.append(geom) + + logger.debug(f"---> Geometry array: {geoms}") + + return geoms From fa0bd08e8823f06bbd2d237bff4fec87b2fdf64a Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 13 Sep 2024 15:58:51 +0800 Subject: [PATCH 013/185] Add code to handle geometry fields for OCR bulk import. The importer will create a model instance for each feature in the geojson feature collection. --- boranga/components/occurrence/models.py | 115 ++++++++++++++++++++---- 1 file changed, 99 insertions(+), 16 deletions(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 9976de64..199a6705 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -5437,7 +5437,9 @@ def process(self): self.datetime_error = timezone.now() self.error_message = "Errors occurred during processing:\n" for error in errors: - self.error_message += error["error_message"] + "\n" + self.error_message += ( + f"Row: {error['row_index'] + 1}. Error: {error['error_message']}\n" + ) else: # Set the task to completed self.processing_status = ( @@ -5471,18 +5473,27 @@ def process_row(self, index, row, errors): total_column_error_count = 0 models = {} - + geometries = {} # Validate each cell for index, column in enumerate(self.schema.columns.all()): - # logger.debug(f" Processing column: {column}") - - # logger.debug(f" Cell value: {row[index]}") column_error_count = 0 cell_value = row[index] cell_value, errors_added = column.validate(cell_value, index, errors) + # Special case for geojson feature collection + model_class = apps.get_model( + "boranga", column.django_import_content_type.model + ) + if ( + issubclass(model_class, GeometryBase) + and type(cell_value) is list + and len(cell_value) > 0 + ): + geometries[model_class._meta.model_name] = cell_value + cell_value = geometries[model_class._meta.model_name][0] + column_error_count += errors_added row_error_count += column_error_count @@ -5526,6 +5537,7 @@ def process_row(self, index, row, errors): "boranga", current_model_name, ) + current_model_instance = model_class(**model_data) logger.debug( @@ -5555,6 +5567,9 @@ def process_row(self, index, row, errors): for potential_parent_model_key in [ m for m in models if m != current_model_name ]: + logger.debug( + f"Checking if {current_model_name} has a relationship with {potential_parent_model_key}" + ) # Check if this model has a relationship with the current model potential_parent_instance = model_instances[ potential_parent_model_key @@ -5616,7 +5631,21 @@ def process_row(self, index, row, errors): model_instances[current_model_instance._meta.model_name] = ( current_model_instance ) - logger.debug(f"Model instance created: {current_model_instance}") + logger.info(f"Model instance created: {current_model_instance}") + + # Deal with special case of creating mutliple geometries based on the + # geojson text from the column + if current_model_instance._meta.model_name in geometries: + logger.info( + f"Creating multiple geometries for {current_model_instance}" + ) + for geometry in geometries[current_model_instance._meta.model_name][ + 1: + ]: + current_model_instance.pk = None + current_model_instance.geometry = geometry + current_model_instance.save() + except IntegrityError as e: logger.error(f"Error creating model instance: {e}") errors.append( @@ -5723,7 +5752,6 @@ def preview_import_file(self): f"Model {model_class} does not have field {column.django_import_field_name}" ) model_field = model_class._meta.get_field(column.django_import_field_name) - logger.debug(f"model_field_type: {type(model_field)}") dv = None if column.default_value is not None: dv = DataValidation( @@ -6085,6 +6113,8 @@ def preview_foreign_key_values_xlsx(self): return workbook def validate(self, cell_value, index, errors): + from boranga.components.spatial.utils import get_geometry_array_from_geojson + errors_added = 0 model_class = apps.get_model("boranga", self.django_import_content_type.model) @@ -6142,7 +6172,7 @@ def validate(self, cell_value, index, errors): "row_index": index, "error_type": "column", "data": cell_value, - "error_message": f"Value {cell_value} in column {self.column_header_name} is not a string", + "error_message": f"Value {cell_value} in column {self.xlsx_column_header_name} is not a string", } ) errors_added += 1 @@ -6164,7 +6194,7 @@ def validate(self, cell_value, index, errors): logger.debug(f"display_value: '{display_value}'") logger.debug([choice[1] for choice in choices]) if display_value not in [choice[1] for choice in choices]: - errors_message = ( + error_message = ( f"Value '{display_value}' in column {self.xlsx_column_header_name} " "is not in the list" ) @@ -6173,7 +6203,7 @@ def validate(self, cell_value, index, errors): "row_index": index, "error_type": "column", "data": cell_value, - "error_message": errors_message, + "error_message": error_message, } ) errors_added += 1 @@ -6187,6 +6217,53 @@ def validate(self, cell_value, index, errors): ) return cell_value, errors_added + if isinstance(field, gis_models.GeometryField): + try: + geom_json = json.loads(cell_value) + except json.JSONDecodeError: + error_message = f"Value {cell_value} in column {self.xlsx_column_header_name} is not a valid JSON" + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": error_message, + } + ) + errors_added += 1 + return cell_value, errors_added + + cell_value = [] + + geojson_type = geom_json.get("type", None) + if not geojson_type or geojson_type != "FeatureCollection": + error_message = ( + f"Value {cell_value} in column {self.xlsx_column_header_name} " + "does not contain a valid FeatureCollection" + ) + errors.append( + { + "row_index": index, + "error_type": "column", + "data": cell_value, + "error_message": error_message, + } + ) + errors_added += 1 + + return cell_value, errors_added + + cell_value = get_geometry_array_from_geojson( + geom_json, + cell_value, + index, + self.xlsx_column_header_name, + errors, + errors_added, + ) + + return cell_value, errors_added + if xlsx_data_validation_type == "textLength" and field.max_length: if len(str(cell_value)) > field.max_length: error_message = f"Value {cell_value} in column {self.xlsx_column_header_name} has too many characters" @@ -6202,12 +6279,13 @@ def validate(self, cell_value, index, errors): if xlsx_data_validation_type == "whole": if not isinstance(cell_value, int): + errors_message = f"Value {cell_value} in column {self.xlsx_column_header_name} is not an integer" errors.append( { "row_index": index, "error_type": "column", "data": cell_value, - "error_message": f"Value {cell_value} in column {self.column_header_name} is not an integer", + "error_message": errors_message, } ) errors_added += 1 @@ -6216,12 +6294,13 @@ def validate(self, cell_value, index, errors): try: cell_value = Decimal(cell_value) except Exception: + error_message = f"Value {cell_value} in column {self.xlsx_column_header_name} is not a decimal" errors.append( { "row_index": index, "error_type": "column", "data": cell_value, - "error_message": f"Value {cell_value} in column {self.column_header_name} is not a decimal", + "error_message": error_message, } ) errors_added += 1 @@ -6235,7 +6314,7 @@ def validate(self, cell_value, index, errors): "row_index": index, "error_type": "column", "data": cell_value, - "error_message": f"Value {cell_value} in column {self.column_header_name} is not a date", + "error_message": f"Value {cell_value} in column {self.xlsx_column_header_name} is not a date", } ) errors_added += 1 @@ -6249,7 +6328,7 @@ def validate(self, cell_value, index, errors): "row_index": index, "error_type": "column", "data": cell_value, - "error_message": f"Value {cell_value} in column {self.column_header_name} is not a time", + "error_message": f"Value {cell_value} in column {self.xlsx_column_header_name} is not a time", } ) errors_added += 1 @@ -6282,7 +6361,7 @@ def validate(self, cell_value, index, errors): error_message = ( f"Can't find {self.django_import_field_name} record by looking up " f"{self.django_lookup_field_name} with value {cell_value} " - f"for column {self.column_header_name}" + f"for column {self.xlsx_column_header_name}" ) errors.append( { @@ -6301,12 +6380,16 @@ def validate(self, cell_value, index, errors): if xlsx_data_validation_type == "list": if cell_value not in self.xlsx_data_validation_formula1: + error_message = ( + f"Value {cell_value} in column {self.xlsx_column_header_name} " + "is not in the list" + ) errors.append( { "row_index": index, "error_type": "column", "data": cell_value, - "error_message": f"Value {cell_value} in column {self.column_header_name} is not in the list", + "error_message": error_message, } ) errors_added += 1 From cae15b80a1f8d50aeed9ba0951bcbce3703668ba Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 10:00:40 +0800 Subject: [PATCH 014/185] Modify isReadOnly computed to include other finalised statuses. Modify conservation_list_proposed computed so that 'Proposed' doesn't show up for other finalised statuses. --- .../common/conservation_status/community_status.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue b/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue index db08d555..409a5127 100644 --- a/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue +++ b/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue @@ -607,7 +607,7 @@ export default { ) { return true; } else { - if (this.conservation_status_obj.processing_status == "Ready For Agenda") { + if (["Ready For Agenda", "Approved", "Closed", "DeListed", "Discarded"].includes(this.conservation_status_obj.processing_status)) { return true; } if ( @@ -635,7 +635,7 @@ export default { } }, conservation_list_proposed: function () { - return !(this.conservation_status_obj.processing_status == "Approved" || this.conservation_status_obj.processing_status == "DeListed") + return !(['Approved', 'DeListed', 'Declined', 'Closed', 'Unlocked'].includes(this.conservation_status_obj.processing_status)) }, canViewCurrentList: function () { return (this.conservation_status_obj.processing_status == "Approved" || this.conservation_status_obj.processing_status == "DeListed") ? false : true; From e9bc0984baa5cf60fce977982826e1dbd3ea6c14 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 10:12:43 +0800 Subject: [PATCH 015/185] Remove geojson_point, geojson_polygon fields from OCRLocation model and remove related references. --- boranga/components/occurrence/models.py | 4 - boranga/components/occurrence/serializers.py | 14 - .../components/common/component_map_test.vue | 329 ------------------ .../conservation_status/species_status.vue | 3 +- ...move_occlocation_geojson_point_and_more.py | 29 ++ 5 files changed, 31 insertions(+), 348 deletions(-) delete mode 100644 boranga/frontend/boranga/src/components/common/component_map_test.vue create mode 100644 boranga/migrations/0460_remove_occlocation_geojson_point_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 199a6705..4548e064 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1888,8 +1888,6 @@ class OCRLocation(models.Model): location_accuracy = models.ForeignKey( LocationAccuracy, on_delete=models.SET_NULL, null=True, blank=True ) - geojson_point = gis_models.PointField(srid=4326, blank=True, null=True) - geojson_polygon = gis_models.PolygonField(srid=4326, blank=True, null=True) region = models.ForeignKey( Region, default=None, on_delete=models.CASCADE, null=True, blank=True @@ -4191,8 +4189,6 @@ class OCCLocation(models.Model): location_accuracy = models.ForeignKey( LocationAccuracy, on_delete=models.SET_NULL, null=True, blank=True ) - geojson_point = gis_models.PointField(srid=4326, blank=True, null=True) - geojson_polygon = gis_models.PolygonField(srid=4326, blank=True, null=True) region = models.ForeignKey( Region, default=None, on_delete=models.CASCADE, null=True, blank=True diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index 4b3277f2..e3ddffd9 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -806,20 +806,6 @@ def get_has_points(self, obj): .exists() ) - # def get_geojson_point(self,obj): - # if(obj.geojson_point): - # coordinates = GEOSGeometry(obj.geojson_point).coords - # return coordinates - # else: - # return None - - # def get_geojson_polygon(self,obj): - # if(obj.geojson_polygon): - # coordinates = GEOSGeometry(obj.geojson_polygon).coords - # return coordinates - # else: - # return None - class BaseTypeSerializer(serializers.Serializer): model_class = serializers.SerializerMethodField() diff --git a/boranga/frontend/boranga/src/components/common/component_map_test.vue b/boranga/frontend/boranga/src/components/common/component_map_test.vue deleted file mode 100644 index 257d638a..00000000 --- a/boranga/frontend/boranga/src/components/common/component_map_test.vue +++ /dev/null @@ -1,329 +0,0 @@ - - - - - - diff --git a/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue b/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue index 51609115..ff51b9a4 100644 --- a/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue +++ b/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue @@ -182,6 +182,7 @@ @@ -628,7 +629,7 @@ export default { return true; }, conservation_list_proposed: function () { - return !(this.conservation_status_obj.processing_status == "Approved" || this.conservation_status_obj.processing_status == "DeListed") + return !(['Approved', 'DeListed', 'Declined', 'Closed', 'Unlocked'].includes(this.conservation_status_obj.processing_status)) }, canViewCurrentList: function () { return (this.conservation_status_obj.processing_status == "Approved" || this.conservation_status_obj.processing_status == "DeListed") ? false : true; diff --git a/boranga/migrations/0460_remove_occlocation_geojson_point_and_more.py b/boranga/migrations/0460_remove_occlocation_geojson_point_and_more.py new file mode 100644 index 00000000..eddfdf48 --- /dev/null +++ b/boranga/migrations/0460_remove_occlocation_geojson_point_and_more.py @@ -0,0 +1,29 @@ +# Generated by Django 5.0.9 on 2024-09-16 02:11 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0459_alter_communitydocument__file_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="occlocation", + name="geojson_point", + ), + migrations.RemoveField( + model_name="occlocation", + name="geojson_polygon", + ), + migrations.RemoveField( + model_name="ocrlocation", + name="geojson_point", + ), + migrations.RemoveField( + model_name="ocrlocation", + name="geojson_polygon", + ), + ] From 6040a923cc57be6314474a120f34fca4d95c9ea6 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 10:18:51 +0800 Subject: [PATCH 016/185] Remove unused initial_data.json. --- boranga/data/initial_data.json | 239 --------------------------------- 1 file changed, 239 deletions(-) delete mode 100755 boranga/data/initial_data.json diff --git a/boranga/data/initial_data.json b/boranga/data/initial_data.json deleted file mode 100755 index d19c6a1d..00000000 --- a/boranga/data/initial_data.json +++ /dev/null @@ -1,239 +0,0 @@ -[ -{ - "model": "boranga.applicationtype", - "pk": 1, - "fields": { - "name": "T Class", - "order": 1, - "visible": true - } -}, -{ - "model": "boranga.applicationtype", - "pk": 2, - "fields": { - "name": "Filming", - "order": 2, - "visible": true - } -}, -{ - "model": "boranga.applicationtype", - "pk": 3, - "fields": { - "name": "Event", - "order": 3, - "visible": true - } -}, -{ - "model": "boranga.activitymatrix", - "pk": 1, - "fields": { - "name": "Commercial Operator", - "description": "testing", - "schema": [ - {} - ], - "replaced_by": null, - "version": 1, - "ordered": false - } -}, -{ - "model": "boranga.proposaltype", - "pk": 1, - "fields": { - "description": "T Class Dummy", - "name": "T Class", - "schema": [ - {} - ], - "replaced_by": null, - "version": 1 - } -}, -{ - "model": "boranga.proposalassessorgroup", - "pk": 1, - "fields": { - "name": "Default Group", - "region": null, - "default": true, - "members": [ - 255 - ] - } -}, -{ - "model": "boranga.proposalapprovergroup", - "pk": 1, - "fields": { - "name": "Default Group", - "region": null, - "default": true, - "members": [ - 255 - ] - } -}, -{ - "model": "boranga.proposal", - "pk": 1, - "fields": { - "proposal_type": "new_proposal", - "data": [ - { - "regionActivitySection": [ - { - "Sub-activity level 2": "", - "Sub-activity level 1": "", - "District": null, - "Management area": "", - "Region": null, - "ActivityType": "" - } - ] - } - ], - "assessor_data": null, - "comment_data": null, - "schema": [ - {} - ], - "proposed_issuance_approval": null, - "customer_status": "draft", - "applicant": 1, - "lodgement_number": "P000001", - "lodgement_sequence": 0, - "lodgement_date": null, - "proxy_applicant": null, - "submitter": 255, - "assigned_officer": null, - "assigned_approver": null, - "processing_status": "draft", - "id_check_status": "not_checked", - "compliance_check_status": "not_checked", - "character_check_status": "not_checked", - "review_status": "not_reviewed", - "approval": null, - "previous_application": null, - "proposed_decline_status": false, - "title": null, - "activity": "", - "tenure": null, - "region": null, - "district": null, - "application_type": 1, - "approval_level": "", - "approval_level_document": null, - "approval_comment": "" - } -}, -{ - "model": "boranga.proposal", - "pk": 2, - "fields": { - "proposal_type": "new_proposal", - "data": [ - { - "regionActivitySection": [ - { - "Sub-activity level 2": "", - "Sub-activity level 1": "", - "District": null, - "Management area": "", - "Region": null, - "ActivityType": "" - } - ] - } - ], - "assessor_data": null, - "comment_data": null, - "schema": [ - {} - ], - "proposed_issuance_approval": null, - "customer_status": "draft", - "applicant": 1, - "lodgement_number": "P000002", - "lodgement_sequence": 0, - "lodgement_date": null, - "proxy_applicant": null, - "submitter": 255, - "assigned_officer": null, - "assigned_approver": null, - "processing_status": "draft", - "id_check_status": "not_checked", - "compliance_check_status": "not_checked", - "character_check_status": "not_checked", - "review_status": "not_reviewed", - "approval": null, - "previous_application": null, - "proposed_decline_status": false, - "title": null, - "activity": "", - "tenure": null, - "region": null, - "district": null, - "application_type": 1, - "approval_level": "", - "approval_level_document": null, - "approval_comment": "" - } -}, -{ - "model": "boranga.proposal", - "pk": 3, - "fields": { - "proposal_type": "new_proposal", - "data": [ - { - "regionActivitySection": [ - { - "Sub-activity level 2": "", - "Sub-activity level 1": "", - "District": null, - "Management area": "", - "Region": null, - "ActivityType": "" - } - ] - } - ], - "assessor_data": null, - "comment_data": null, - "schema": [ - {} - ], - "proposed_issuance_approval": null, - "customer_status": "draft", - "applicant": 1, - "lodgement_number": "P000003", - "lodgement_sequence": 0, - "lodgement_date": null, - "proxy_applicant": null, - "submitter": 255, - "assigned_officer": null, - "assigned_approver": null, - "processing_status": "draft", - "id_check_status": "not_checked", - "compliance_check_status": "not_checked", - "character_check_status": "not_checked", - "review_status": "not_reviewed", - "approval": null, - "previous_application": null, - "proposed_decline_status": false, - "title": null, - "activity": "", - "tenure": null, - "region": null, - "district": null, - "application_type": 1, - "approval_level": "", - "approval_level_document": null, - "approval_comment": "" - } -} -] From 0ba16c1ea49e5df8eabb27496766abf41841efd2 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 10:23:22 +0800 Subject: [PATCH 017/185] Task 6619: Rename Actions as Action Logs and Communications as Communication Logs. --- boranga/frontend/boranga/src/components/common/comms_logs.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boranga/frontend/boranga/src/components/common/comms_logs.vue b/boranga/frontend/boranga/src/components/common/comms_logs.vue index 1432a5d8..1ae9f22a 100755 --- a/boranga/frontend/boranga/src/components/common/comms_logs.vue +++ b/boranga/frontend/boranga/src/components/common/comms_logs.vue @@ -4,7 +4,7 @@
Logs
- +
View @@ -18,7 +18,7 @@
- +
View From 9cf4e145e7b97080db6905f9f096c3785edc3a99 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 10:28:06 +0800 Subject: [PATCH 018/185] Task 6613R: emove the validation that doesn't allow a meeting to be scheduled if there are no agenda items. --- .../boranga/src/components/internal/meetings/meeting.vue | 3 --- 1 file changed, 3 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/meetings/meeting.vue b/boranga/frontend/boranga/src/components/internal/meetings/meeting.vue index f9712881..c624c661 100644 --- a/boranga/frontend/boranga/src/components/internal/meetings/meeting.vue +++ b/boranga/frontend/boranga/src/components/internal/meetings/meeting.vue @@ -372,9 +372,6 @@ export default { blank_fields.push('Please select at least two committee members who will be attending'); } } - if (vm.$refs.cs_queue.$refs.cs_queue_datatable.vmDataTable.rows().count() == 0) { - blank_fields.push(' Please add at least one Agenda record') - } if (vm.meeting_obj.location_id == null || vm.meeting_obj.location_id == '') { blank_fields.push(' Please select Location') } From d3bc3f7ab24a0c5e9cd79e3cdb6cb24ae2a7739f Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 11:13:43 +0800 Subject: [PATCH 019/185] Bug fix: Prevent the form submissions getting stuck when the user clicks the save and exit button before the map is loaded (in which case the layer is not ready and the getFeatures call fails). --- .../boranga/src/components/common/component_map.vue | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/boranga/frontend/boranga/src/components/common/component_map.vue b/boranga/frontend/boranga/src/components/common/component_map.vue index 33bd83ef..3fba787c 100644 --- a/boranga/frontend/boranga/src/components/common/component_map.vue +++ b/boranga/frontend/boranga/src/components/common/component_map.vue @@ -4580,6 +4580,15 @@ export default { layer_name = this.defaultQueryLayerName; } const format = new GeoJSON(); + if (!this.layerSources[layer_name]) { + // Just adding this to cover the case of the user quickly + // pressing save and exit button before the map is fully loaded + // and the layer sources are not yet available + console.error( + `Layer source ${layer_name} not found. Cannot get features.` + ); + return; + } const layerFeatures = this.layerSources[layer_name].getFeatures(); const features = []; From ff514efffb1481f48de1145ef594b3d9c1ad4715 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 11:52:22 +0800 Subject: [PATCH 020/185] Make RelatedItem object hashable (set we can use set(list( to remvoe duplicates easily. --- boranga/components/main/related_item.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/boranga/components/main/related_item.py b/boranga/components/main/related_item.py index e9426d13..6ff7c630 100644 --- a/boranga/components/main/related_item.py +++ b/boranga/components/main/related_item.py @@ -11,6 +11,29 @@ def __init__( self.status = status self.action_url = action_url + def __hash__(self): + return hash( + ( + self.model_name, + self.identifier, + self.descriptor, + self.status, + self.action_url, + ) + ) + + def __eq__(self, other): + return ( + self.identifier == other.identifier + and self.model_name == other.model_name + and self.descriptor == other.descriptor + and self.status == other.status + and self.action_url == other.action_url + ) + + def __str__(self): + return f"{self.identifier}" + class RelatedItemsSerializer(serializers.Serializer): model_name = serializers.CharField() From 617b4e0099855d31f489708988d69beec6759feb Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 11:55:15 +0800 Subject: [PATCH 021/185] Remove duplicates that were being returend from get_related_items method on Community model. --- boranga/components/species_and_communities/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/boranga/components/species_and_communities/models.py b/boranga/components/species_and_communities/models.py index 3ec61a43..7fdda315 100644 --- a/boranga/components/species_and_communities/models.py +++ b/boranga/components/species_and_communities/models.py @@ -1518,6 +1518,9 @@ def get_related_items(self, filter_type, **kwargs): ) ) + # Remove duplicates + return_list = list(set(return_list)) + return return_list @property From 34884ca0d0f93f337c5611fcdd31ba692ae8a4e7 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 13:52:14 +0800 Subject: [PATCH 022/185] Modify BaseConservationStatusSerializer priority list and category fields to return the code and label (if exists) so that they match the values in the drop downs on the front end (i.e. code and label rather than just code). --- .../conservation_status/serializers.py | 73 +++++++++++++++---- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/boranga/components/conservation_status/serializers.py b/boranga/components/conservation_status/serializers.py index 0fa1f03d..1fb55e94 100644 --- a/boranga/components/conservation_status/serializers.py +++ b/boranga/components/conservation_status/serializers.py @@ -530,21 +530,11 @@ class BaseConservationStatusSerializer(serializers.ModelSerializer): group_type = serializers.SerializerMethodField(read_only=True) group_type_id = serializers.SerializerMethodField(read_only=True) is_submitter = serializers.SerializerMethodField(read_only=True) - wa_legislative_list = serializers.CharField( - source="wa_legislative_list.code", read_only=True, allow_null=True - ) - wa_legislative_category = serializers.CharField( - source="wa_legislative_category.code", read_only=True, allow_null=True - ) - wa_priority_list = serializers.CharField( - source="wa_priority_list.code", read_only=True, allow_null=True - ) - wa_priority_category = serializers.CharField( - source="wa_priority_category.code", read_only=True, allow_null=True - ) - commonwealth_conservation_list = serializers.CharField( - source="commonwealth_conservation_list.code", read_only=True, allow_null=True - ) + wa_legislative_list = serializers.SerializerMethodField(read_only=True) + wa_legislative_category = serializers.SerializerMethodField(read_only=True) + wa_priority_list = serializers.SerializerMethodField(read_only=True) + wa_priority_category = serializers.SerializerMethodField(read_only=True) + commonwealth_conservation_list = serializers.SerializerMethodField(read_only=True) class Meta: model = ConservationStatus @@ -598,6 +588,54 @@ class Meta: "is_submitter", ) + def get_wa_legislative_list(self, obj): + if not obj.wa_legislative_list: + return None + + if obj.wa_legislative_list.code and obj.wa_legislative_list.label: + return f"{obj.wa_legislative_list.code} - {obj.wa_legislative_list.label}" + + return obj.wa_legislative_list.code + + def get_wa_legislative_category(self, obj): + if not obj.wa_legislative_category: + return None + + if obj.wa_legislative_category.code and obj.wa_legislative_category.label: + return f"{obj.wa_legislative_category.code} - {obj.wa_legislative_category.label}" + + return obj.wa_legislative_category.code + + def get_wa_priority_list(self, obj): + if not obj.wa_priority_list: + return None + + if obj.wa_priority_list.code and obj.wa_priority_list.label: + return f"{obj.wa_priority_list.code} - {obj.wa_priority_list.label}" + + return obj.wa_priority_list.code + + def get_wa_priority_category(self, obj): + if not obj.wa_priority_category: + return None + + if obj.wa_priority_category.code and obj.wa_priority_category.label: + return f"{obj.wa_priority_category.code} - {obj.wa_priority_category.label}" + + return obj.wa_priority_category.code + + def get_commonwealth_conservation_list(self, obj): + if not obj.commonwealth_conservation_list: + return None + + if ( + obj.commonwealth_conservation_list.code + and obj.commonwealth_conservation_list.label + ): + return f"{obj.commonwealth_conservation_list.code} - {obj.commonwealth_conservation_list.label}" + + return obj.commonwealth_conservation_list.code + def get_readonly(self, obj): return False @@ -789,10 +827,15 @@ class Meta: "community_id", "conservation_status_number", "wa_legislative_list_id", + "wa_legislative_list", "wa_legislative_category_id", + "wa_legislative_category", "wa_priority_list_id", + "wa_priority_list", "wa_priority_category_id", + "wa_priority_category", "commonwealth_conservation_list_id", + "commonwealth_conservation_list", "international_conservation", "conservation_criteria", "recommended_conservation_criteria", From 8993b7935a4e7d915b4f26a01013bf2374d52601 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 13:53:16 +0800 Subject: [PATCH 023/185] Bug fix: Make sure legislative list / category information still displays when fields are readonly otherwise display N/A. --- .../conservation_status/community_status.vue | 24 +++++++++--------- .../conservation_status/species_status.vue | 25 +++++++++---------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue b/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue index 409a5127..a52eee79 100644 --- a/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue +++ b/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue @@ -174,8 +174,8 @@
@@ -206,8 +206,8 @@
@@ -237,8 +237,8 @@ @@ -268,8 +268,8 @@ @@ -302,8 +302,8 @@ @@ -313,7 +313,7 @@
@@ -323,7 +323,7 @@
diff --git a/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue b/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue index ff51b9a4..eb178cda 100644 --- a/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue +++ b/boranga/frontend/boranga/src/components/common/conservation_status/species_status.vue @@ -182,9 +182,8 @@ @@ -215,8 +214,8 @@ @@ -246,8 +245,8 @@ @@ -277,8 +276,8 @@ @@ -311,8 +310,8 @@ @@ -322,7 +321,7 @@
@@ -332,7 +331,7 @@
From c9e5964e8af493cffe23281c931673762e5a505c Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 16:06:20 +0800 Subject: [PATCH 024/185] Update send back to assessor email template text. --- .../cs_proposals/send_approver_sendback_notification.html | 6 +++--- .../cs_proposals/send_approver_sendback_notification.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.html b/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.html index ff594f58..622db4fd 100644 --- a/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.html +++ b/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.html @@ -1,13 +1,13 @@ {% extends 'boranga/emails/base_email.html' %} {% block content %} - The Proposal {{ cs_proposal.conservation_status_number }} has been sent back by approver. + The Proposal {{ cs_proposal.conservation_status_number }} has been sent back by the approver. -

Approver comments: {{ approver_comment }}

+

Reason / Comments: {{ approver_comment }}

You can access this Proposal using the following link:

Access Proposal -{% endblock %} +{% endblock %} diff --git a/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.txt b/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.txt index 15e3d817..52ed3a92 100644 --- a/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.txt +++ b/boranga/components/conservation_status/templates/boranga/emails/cs_proposals/send_approver_sendback_notification.txt @@ -1,11 +1,11 @@ {% extends 'boranga/emails/base_email.txt' %} {% block content %} - The Proposal {{ cs_proposal.conservation_status_number }} has been sent back by approver. + The Proposal {{ cs_proposal.conservation_status_number }} has been sent back by the approver. - Approver comments: {{ approver_comment }} + Reason / Comments: {{ approver_comment }} You can access this Proposal using the following link: {{url}} -{% endblock %} +{% endblock %} From 806396a6b69bb4bca18a4cb4ba931746f400347e Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 16:07:09 +0800 Subject: [PATCH 025/185] Bug fix: Modify move_to_status method so that email sends with approver comment when approver sends back to assessor. --- .../components/conservation_status/models.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/boranga/components/conservation_status/models.py b/boranga/components/conservation_status/models.py index fa52f0f2..3bd2abbf 100644 --- a/boranga/components/conservation_status/models.py +++ b/boranga/components/conservation_status/models.py @@ -965,22 +965,18 @@ def can_user_assign_to_self(self, request): def has_assessor_mode(self, request): status_without_assessor = [ + ConservationStatus.PROCESSING_STATUS_APPROVED, ConservationStatus.PROCESSING_STATUS_WITH_APPROVER, ConservationStatus.PROCESSING_STATUS_CLOSED, ConservationStatus.PROCESSING_STATUS_DECLINED, + ConservationStatus.PROCESSING_STATUS_DELISTED, ConservationStatus.PROCESSING_STATUS_DRAFT, ] if self.processing_status in status_without_assessor: - if self.processing_status in [ - ConservationStatus.PROCESSING_STATUS_UNLOCKED, - ]: - return is_conservation_status_approver(request) - return False - elif self.processing_status == ConservationStatus.PROCESSING_STATUS_APPROVED: - return is_conservation_status_assessor( - request - ) or is_conservation_status_approver(request) + + elif self.processing_status == ConservationStatus.PROCESSING_STATUS_UNLOCKED: + return is_conservation_status_approver(request) else: if not self.assigned_officer: return False @@ -1216,12 +1212,16 @@ def move_to_status(self, request, status, approver_comment): if self.processing_status == status: return - if self.processing_status == ConservationStatus.PROCESSING_STATUS_WITH_APPROVER: + if ( + self.processing_status + == ConservationStatus.PROCESSING_STATUS_READY_FOR_AGENDA + ): self.approver_comment = "" if approver_comment: self.approver_comment = approver_comment self.save() send_proposal_approver_sendback_email_notification(request, self) + previous_status = self.processing_status self.processing_status = status self.save() From 5c912e98a0a74a3f60c84945841c94f0a69a1817 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 16:07:34 +0800 Subject: [PATCH 026/185] Add back to assessor modal component. --- .../conservation_status/back_to_assessor.vue | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 boranga/frontend/boranga/src/components/internal/conservation_status/back_to_assessor.vue diff --git a/boranga/frontend/boranga/src/components/internal/conservation_status/back_to_assessor.vue b/boranga/frontend/boranga/src/components/internal/conservation_status/back_to_assessor.vue new file mode 100644 index 00000000..f27038fa --- /dev/null +++ b/boranga/frontend/boranga/src/components/internal/conservation_status/back_to_assessor.vue @@ -0,0 +1,103 @@ + + + From eb8cc50293cfb4d241f53be132ecead18b4bccef Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 16 Sep 2024 16:09:05 +0800 Subject: [PATCH 027/185] Modify conditions under which change type and effective / review dates can be edited. --- .../common/conservation_status/community_status.vue | 10 +++++----- .../common/conservation_status/species_status.vue | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue b/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue index a52eee79..1963716f 100644 --- a/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue +++ b/boranga/frontend/boranga/src/components/common/conservation_status/community_status.vue @@ -76,7 +76,7 @@
-