From a76457f32692b5275bf6faab5da438d9653a1667 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 09:49:17 +0800 Subject: [PATCH 01/47] Add new bulk import component (front end only at this stage). --- .../internal/occurrence/bulk_import.vue | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue new file mode 100644 index 00000000..59b7782d --- /dev/null +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue @@ -0,0 +1,211 @@ + + + + From ee80879df781c9884cb7bf250b46059824ff9229 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 09:49:33 +0800 Subject: [PATCH 02/47] Add route for new bulk import component. --- .../boranga/src/components/internal/routes/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/boranga/frontend/boranga/src/components/internal/routes/index.js b/boranga/frontend/boranga/src/components/internal/routes/index.js index ab3dc67b..bb30094d 100755 --- a/boranga/frontend/boranga/src/components/internal/routes/index.js +++ b/boranga/frontend/boranga/src/components/internal/routes/index.js @@ -10,6 +10,7 @@ import OccurrenceDash from '../occurrence/dashboard.vue' import Occurrence from '../occurrence/occurrence.vue' import OccurrenceReport from '../occurrence/occurrence_report.vue' import OccurrenceReportReferral from '../occurrence/referral.vue' +import BulkImport from '../occurrence/bulk_import.vue' export default { @@ -63,6 +64,11 @@ export default }, }, children: [ + { + path: 'bulk_import/', + name: "occurrence-report-bulk-import", + component: BulkImport + }, { path: ':occurrence_report_id', component: { From 57d23d27a5c0f5389f347a5b47461f32fdd1c2dd Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 09:50:40 +0800 Subject: [PATCH 03/47] Add bulk import button above OCR datatable. --- .../common/occurrence_report_community_dashboard.vue | 2 ++ .../components/common/occurrence_report_fauna_dashboard.vue | 2 ++ .../components/common/occurrence_report_flora_dashboard.vue | 3 +++ 3 files changed, 7 insertions(+) diff --git a/boranga/frontend/boranga/src/components/common/occurrence_report_community_dashboard.vue b/boranga/frontend/boranga/src/components/common/occurrence_report_community_dashboard.vue index d87e4435..03e1cc2a 100644 --- a/boranga/frontend/boranga/src/components/common/occurrence_report_community_dashboard.vue +++ b/boranga/frontend/boranga/src/components/common/occurrence_report_community_dashboard.vue @@ -60,6 +60,8 @@
+ Bulk Import
diff --git a/boranga/frontend/boranga/src/components/common/occurrence_report_fauna_dashboard.vue b/boranga/frontend/boranga/src/components/common/occurrence_report_fauna_dashboard.vue index 92e9a9a8..672b90ec 100644 --- a/boranga/frontend/boranga/src/components/common/occurrence_report_fauna_dashboard.vue +++ b/boranga/frontend/boranga/src/components/common/occurrence_report_fauna_dashboard.vue @@ -60,6 +60,8 @@
+ Bulk Import
diff --git a/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue b/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue index c556b7b0..1b6c8713 100644 --- a/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue +++ b/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue @@ -60,6 +60,9 @@
+ Bulk Import +
From 264d05ef5c4f296ec704a2254fb4f52255419e25 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 10:47:44 +0800 Subject: [PATCH 04/47] Add OccurrenceReportBulkImportTask. --- boranga/components/occurrence/models.py | 95 +++++++++++++++++++ ...occurrencereportbulkimporttask_and_more.py | 70 ++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 boranga/migrations/0428_occurrencereportbulkimporttask_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 799e77c2..12e4a9a3 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -5225,3 +5225,98 @@ def save(self, *args, **kwargs): "identification", ], ) + + +def get_occurrence_report_bulk_import_path(instance, filename): + return ( + f"occurrence_report/bulk-imports/{instance.occurrence_report.occurrence_report_number}" + f"/{instance.datetime_queued}/{filename}" + ) + + +class OccurrenceReportBulkImportTask(models.Model): + _file = models.FileField( + upload_to=get_occurrence_report_bulk_import_path, + max_length=512, + storage=private_storage, + ) + rows = models.IntegerField(null=True) + rows_processed = models.IntegerField(default=0) + + datetime_queued = models.DateTimeField(auto_now_add=True) + datetime_started = models.DateTimeField(null=True, blank=True) + datetime_completed = models.DateTimeField(null=True, blank=True) + + datetime_error = models.DateTimeField(null=True, blank=True) + error_row = models.IntegerField(null=True, blank=True) + error_message = models.TextField(null=True, blank=True) + + email_user = models.IntegerField(null=False) + + PROCESSING_STATUS_QUEUED = "queued" + PROCESSING_STATUS_STARTED = "started" + PROCESSING_STATUS_FAILED = "failed" + PROCESSING_STATUS_COMPLETED = "completed" + + PROCESSING_STATUS_CHOICES = ( + (PROCESSING_STATUS_QUEUED, "Queued"), + (PROCESSING_STATUS_STARTED, "Started"), + (PROCESSING_STATUS_FAILED, "Failed"), + (PROCESSING_STATUS_COMPLETED, "Completed"), + ) + + processing_status = models.CharField( + max_length=20, + choices=PROCESSING_STATUS_CHOICES, + default=PROCESSING_STATUS_QUEUED, + ) + + class Meta: + app_label = "boranga" + verbose_name = "Occurrence Report Bulk Import Task" + verbose_name_plural = "Occurrence Report Bulk Import Tasks" + + @property + def percentage_complete(self): + if self.rows: + return round((self.rows_processed / self.rows) * 100) + return 0 + + @property + def total_time_taken(self): + if self.datetime_started and self.datetime_completed: + return self.datetime_completed - self.datetime_started + return None + + @property + def time_taken_per_row(self): + if self.datetime_started and self.datetime_completed: + return self.total_time_taken() / self.rows_processed + return None + + @property + def file_size(self): + if self._file: + return self._file.size + return None + + @classmethod + def average_time_taken_per_row(cls): + task_count = cls.objects.filter( + datetime_completed__isnull=False, rows_processed__gt=0 + ).count() + total_time_taken = 0 + for task in cls.objects.filter( + datetime_completed__isnull=False, rows_processed__gt=0 + ): + total_time_taken += task.average_time_taken_per_row() + + return total_time_taken / task_count if task_count > 0 else None + + @property + def estimated_processing_time(self): + if self.rows and self.datetime_queued: + return ( + self.rows - self.rows_processed + ) * OccurrenceReportBulkImportTask.average_time_taken_per_row() + return None diff --git a/boranga/migrations/0428_occurrencereportbulkimporttask_and_more.py b/boranga/migrations/0428_occurrencereportbulkimporttask_and_more.py new file mode 100644 index 00000000..4a82e33b --- /dev/null +++ b/boranga/migrations/0428_occurrencereportbulkimporttask_and_more.py @@ -0,0 +1,70 @@ +# Generated by Django 5.0.8 on 2024-08-15 02:40 + +import boranga.components.occurrence.models +import django.core.files.storage +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0427_alter_community_renamed_from_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="OccurrenceReportBulkImportTask", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "_file", + models.FileField( + max_length=512, + storage=django.core.files.storage.FileSystemStorage( + base_url="/private-media/", + location="/home/oak/dev/boranga/private-media/", + ), + upload_to=boranga.components.occurrence.models.get_occurrence_report_bulk_import_path, + ), + ), + ("rows", models.IntegerField(null=True)), + ("rows_processed", models.IntegerField(default=0)), + ("datetime_queued", models.DateTimeField(auto_now_add=True)), + ("datetime_started", models.DateTimeField(blank=True, null=True)), + ("datetime_completed", models.DateTimeField(blank=True, null=True)), + ("datetime_error", models.DateTimeField(blank=True, null=True)), + ("error_row", models.IntegerField(blank=True, null=True)), + ("error_message", models.TextField(blank=True, null=True)), + ("email_user", models.IntegerField()), + ( + "processing_status", + models.CharField( + choices=[ + ("queued", "Queued"), + ("started", "Started"), + ("failed", "Failed"), + ("completed", "Completed"), + ], + default="queued", + max_length=20, + ), + ), + ], + options={ + "verbose_name": "Occurrence Report Bulk Import Task", + "verbose_name_plural": "Occurrence Report Bulk Import Tasks", + }, + ), + migrations.AlterModelOptions( + name="community", + options={"verbose_name_plural": "communities"}, + ), + ] From 28f8b76ce530c1ad03353f7de867c863f438971b Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:29:30 +0800 Subject: [PATCH 05/47] Add django-filter. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index e4ed9174..337fbe1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ git+https://github.com/xzzy/django-preserialize.git#egg=django-preserialize django-countries~=7.5 django-cron==0.6.0 # This project is no longer maintained django-dynamic-fixture==3.1.3 +django-filter~=24.3 gdal==3.8.4 openpyxl~=3.1 datapackage~=1.15 From 4148ddd3ffb3b778905fc7afed1ddbd2b4e8dd1b Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:29:53 +0800 Subject: [PATCH 06/47] Add OccurrenceReportBulkImportPermission. --- boranga/components/occurrence/permissions.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/boranga/components/occurrence/permissions.py b/boranga/components/occurrence/permissions.py index 17174368..6d975ca2 100644 --- a/boranga/components/occurrence/permissions.py +++ b/boranga/components/occurrence/permissions.py @@ -425,6 +425,17 @@ def has_object_permission(self, request, view, obj): return obj.submitter == request.user.id or is_occurrence_assessor(request) +class OccurrenceReportBulkImportPermission(BasePermission): + def has_permission(self, request, view): + if not request.user.is_authenticated: + return False + + if request.user.is_superuser: + return True + + return is_occurrence_assessor(request) + + class OccurrencePermission(BasePermission): def has_permission(self, request, view): if not request.user.is_authenticated: From 7e1fee07588996a72478526cfc7724c5b4a4928e Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:30:12 +0800 Subject: [PATCH 07/47] Add django_filters to INSTALLED_APPS. --- boranga/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/boranga/settings.py b/boranga/settings.py index abf4e1a6..a3d1af8a 100755 --- a/boranga/settings.py +++ b/boranga/settings.py @@ -128,6 +128,7 @@ def show_toolbar(request): "reversion_compare", "nested_admin", "colorfield", + "django_filters", ] ADD_REVERSION_ADMIN = True From a5559c80bd0346c2583595634380aa00d814dd61 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:30:46 +0800 Subject: [PATCH 08/47] More work on bulk import feature. --- boranga/components/occurrence/models.py | 134 ++++++++++++++++++++++-- 1 file changed, 125 insertions(+), 9 deletions(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 12e4a9a3..914d359c 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1,8 +1,10 @@ import json import logging +import os from abc import abstractmethod from datetime import datetime +import openpyxl import pyproj import reversion from colorfield.fields import ColorField @@ -19,6 +21,7 @@ from django.db import models, transaction from django.db.models import CharField, Count, Func, Q from django.db.models.functions import Cast +from django.utils import timezone from ledger_api_client.ledger_models import EmailUserRO as EmailUser from ledger_api_client.managed_models import SystemGroup from multiselectfield import MultiSelectField @@ -5227,11 +5230,17 @@ def save(self, *args, **kwargs): ) +def validate_bulk_import_file_extension(value): + ext = os.path.splitext(value.name)[1] + valid_extensions = [".xlsx"] + if ext not in valid_extensions: + raise ValidationError( + "Only .xlsx files are supported by the bulk import facility!" + ) + + def get_occurrence_report_bulk_import_path(instance, filename): - return ( - f"occurrence_report/bulk-imports/{instance.occurrence_report.occurrence_report_number}" - f"/{instance.datetime_queued}/{filename}" - ) + return f"occurrence_report/bulk-imports/{timezone.now()}/{filename}" class OccurrenceReportBulkImportTask(models.Model): @@ -5239,8 +5248,9 @@ class OccurrenceReportBulkImportTask(models.Model): upload_to=get_occurrence_report_bulk_import_path, max_length=512, storage=private_storage, + validators=[validate_bulk_import_file_extension], ) - rows = models.IntegerField(null=True) + rows = models.IntegerField(null=True, editable=False) rows_processed = models.IntegerField(default=0) datetime_queued = models.DateTimeField(auto_now_add=True) @@ -5276,6 +5286,10 @@ class Meta: verbose_name = "Occurrence Report Bulk Import Task" verbose_name_plural = "Occurrence Report Bulk Import Tasks" + @property + def file_name(self): + return os.path.basename(self._file.name) + @property def percentage_complete(self): if self.rows: @@ -5295,11 +5309,17 @@ def time_taken_per_row(self): return None @property - def file_size(self): + def file_size_bytes(self): if self._file: return self._file.size return None + @property + def file_size_megabytes(self): + if self.file_size_bytes: + return round(self.file_size_bytes / 1024 / 1024, 2) + return None + @classmethod def average_time_taken_per_row(cls): task_count = cls.objects.filter( @@ -5315,8 +5335,104 @@ def average_time_taken_per_row(cls): @property def estimated_processing_time(self): + average_time_taken_per_row = ( + OccurrenceReportBulkImportTask.average_time_taken_per_row() + ) + if not average_time_taken_per_row: + return "No processing data available to estimate time" if self.rows and self.datetime_queued: - return ( - self.rows - self.rows_processed - ) * OccurrenceReportBulkImportTask.average_time_taken_per_row() + return (self.rows - self.rows_processed) * average_time_taken_per_row return None + + @transaction.atomic + def pre_process(self): + logger.info(f"Pre-queue processing bulk import task {self.id}") + try: + workbook = openpyxl.load_workbook(self._file) + except Exception as e: + logger.error(f"Error opening bulk import file {self._file.name}: {e}") + self.processing_status = ( + OccurrenceReportBulkImportTask.PROCESSING_STATUS_FAILED + ) + self.datetime_error = timezone.now() + self.error_message = f"Error opening bulk import file: {e}" + self.save() + return + + sheet = workbook.active + self.rows = sheet.max_row - 1 + self.save() + + def process(self): + if self.processing_status == self.PROCESSING_STATUS_COMPLETED: + logger.info(f"Bulk import task {self.id} has already been processed") + return + + if self.processing_status == self.PROCESSING_STATUS_FAILED: + logger.info( + f"Bulk import task {self.id} failed. Please correct the issues and try again" + ) + return + + if self.processing_status == self.PROCESSING_STATUS_STARTED: + logger.info(f"Bulk import task {self.id} is already in progress") + return + + self.processing_status = self.PROCESSING_STATUS_STARTED + self.datetime_started = timezone.now() + self.save() + + # Open the file + logger.info(f"Opening bulk import file {self._file.name}") + try: + workbook = openpyxl.load_workbook(self._file) + except Exception as e: + logger.error(f"Error opening bulk import file {self._file.name}: {e}") + self.processing_status = ( + OccurrenceReportBulkImportTask.PROCESSING_STATUS_FAILED + ) + self.datetime_error = timezone.now() + self.error_message = f"Error opening bulk import file: {e}" + self.save() + return + + # Get the first sheet + sheet = workbook.active + + # Get the headers + headers = [cell.value for cell in sheet[1]] + + # Get the rows + rows = list(sheet.iter_rows(min_row=2, values_only=True)) + + # Process the rows + for i, row in enumerate(rows): + self.rows_processed = i + 1 + self.save() + + logger.info(f"Processing row {i + 1}") + try: + self.process_row(row, headers) + except Exception as e: + logger.error(f"Error processing row {i + 1}: {e}") + self.processing_status = ( + OccurrenceReportBulkImportTask.PROCESSING_STATUS_FAILED + ) + self.datetime_error = timezone.now() + self.error_row = i + 1 + self.error_message = f"Error processing row {i + 1}: {e}" + self.save() + return + + # Set the task to completed + self.processing_status = ( + OccurrenceReportBulkImportTask.PROCESSING_STATUS_COMPLETED + ) + self.save() + + logger.info(f"Processing bulk import task {self.id}") + return + + def process_row(self, row, headers): + logger.info(f"Processing row {row}") + return From 1e0e34e73eeb34362cb8fa04dd872e6416857096 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:31:18 +0800 Subject: [PATCH 09/47] Remove unused imports. --- boranga/views.py | 36 +++++++----------------------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/boranga/views.py b/boranga/views.py index 888c0f46..84f8f80a 100644 --- a/boranga/views.py +++ b/boranga/views.py @@ -27,15 +27,11 @@ from boranga.components.species_and_communities.models import Community, Species from boranga.forms import LoginForm from boranga.helpers import ( - is_conservation_status_assessor, is_conservation_status_referee, is_contributor, is_django_admin, is_internal, - is_occurrence_approver, - is_occurrence_assessor, is_occurrence_report_referee, - is_species_communities_approver, ) logger = logging.getLogger(__name__) @@ -209,10 +205,7 @@ def post(self, request): def is_authorised_to_access_community_document(request, document_id): if is_internal(request): # check auth - return ( - request.user.is_superuser - or is_internal(request) - ) + return request.user.is_superuser or is_internal(request) else: return False @@ -220,10 +213,7 @@ def is_authorised_to_access_community_document(request, document_id): def is_authorised_to_access_species_document(request, document_id): if is_internal(request): # check auth - return ( - request.user.is_superuser - or is_internal(request) - ) + return request.user.is_superuser or is_internal(request) else: return False @@ -231,10 +221,7 @@ def is_authorised_to_access_species_document(request, document_id): def is_authorised_to_access_meeting_document(request, document_id): if is_internal(request): # check auth - return ( - request.user.is_superuser - or is_internal(request) - ) + return request.user.is_superuser or is_internal(request) else: return False @@ -257,10 +244,7 @@ def is_authorised_to_access_occurrence_report_document(request, document_id): if is_internal(request): # check auth - return ( - request.user.is_superuser - or is_internal(request) - ) + return request.user.is_superuser or is_internal(request) if is_occurrence_report_referee(request) and is_contributor(request): file_name = get_file_name_from_path(request.path) @@ -322,10 +306,7 @@ def is_authorised_to_access_occurrence_report_document(request, document_id): def is_authorised_to_access_occurrence_document(request, document_id): if is_internal(request): # check auth - return ( - request.user.is_superuser - or is_internal(request) - ) + return request.user.is_superuser or is_internal(request) else: return False @@ -336,10 +317,7 @@ def is_authorised_to_access_conservation_status_document(request, document_id): if is_internal(request): # check auth - return ( - request.user.is_superuser - or is_internal(request) - ) + return request.user.is_superuser or is_internal(request) if is_conservation_status_referee(request) and is_contributor(request): file_name = get_file_name_from_path(request.path) @@ -378,7 +356,7 @@ def is_authorised_to_access_conservation_status_document(request, document_id): document_id, request.path, referee_allowed_paths ) - if is_contributor(request): + if is_contributor(request): contributor_allowed_paths = ["documents", "amendment_request_documents"] file_name = get_file_name_from_path(request.path) return ( From f1ec09a71150caabe8b71f84e930a3a3ceff5c5c Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:31:44 +0800 Subject: [PATCH 10/47] Add OccurrenceReportBulkImportTaskSerializer. --- boranga/components/occurrence/serializers.py | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index 77cc5633..ebff4227 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -36,6 +36,7 @@ OccurrenceReportAmendmentRequest, OccurrenceReportAmendmentRequestDocument, OccurrenceReportApprovalDetails, + OccurrenceReportBulkImportTask, OccurrenceReportDeclinedDetails, OccurrenceReportDocument, OccurrenceReportGeometry, @@ -3857,3 +3858,27 @@ def get_last_updated_by(self, obj): email_user = retrieve_email_user(obj.last_updated_by) return EmailUserSerializer(email_user).data.get("fullname", None) return None + + +class OccurrenceReportBulkImportTaskSerializer(serializers.ModelSerializer): + estimated_processing_time = serializers.CharField(read_only=True) + file_size_megabytes = serializers.CharField(read_only=True) + file_name = serializers.CharField(read_only=True) + + class Meta: + model = OccurrenceReportBulkImportTask + fields = "__all__" + read_only_fields = ( + "id", + "rows", + "rows_processed", + "datetime_queued", + "datetime_started", + "datetime_completed", + "datetime_error", + "error_row", + "error_message", + "processing_status", + "email_user", + "estimated_processing_time", + ) From fc5df65d6f4f486144d34f4287f7e9534c894fb7 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:32:08 +0800 Subject: [PATCH 11/47] Add OccurrenceReportBulkImportTaskViewSet. --- boranga/components/occurrence/api.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/boranga/components/occurrence/api.py b/boranga/components/occurrence/api.py index 152a5e87..3060bcba 100644 --- a/boranga/components/occurrence/api.py +++ b/boranga/components/occurrence/api.py @@ -12,6 +12,7 @@ from django.http import HttpResponse from django.shortcuts import get_object_or_404, redirect from django.urls import reverse +from django_filters import rest_framework as filters from ledger_api_client.ledger_models import EmailUserRO as EmailUser from multiselectfield import MultiSelectField from openpyxl import Workbook @@ -65,6 +66,7 @@ OccurrenceReport, OccurrenceReportAmendmentRequest, OccurrenceReportAmendmentRequestDocument, + OccurrenceReportBulkImportTask, OccurrenceReportDocument, OccurrenceReportGeometry, OccurrenceReportReferral, @@ -110,6 +112,7 @@ IsOccurrenceReportReferee, OccurrenceObjectPermission, OccurrencePermission, + OccurrenceReportBulkImportPermission, OccurrenceReportCopyPermission, OccurrenceReportObjectPermission, OccurrenceReportPermission, @@ -132,6 +135,7 @@ OccurrenceDocumentSerializer, OccurrenceLogEntrySerializer, OccurrenceReportAmendmentRequestSerializer, + OccurrenceReportBulkImportTaskSerializer, OccurrenceReportDocumentSerializer, OccurrenceReportLogEntrySerializer, OccurrenceReportProposalReferralSerializer, @@ -6230,3 +6234,20 @@ def retract(self, request, *args, **kwargs): instance.occurrence_report, context={"request": request} ) return Response(serializer.data, status=status.HTTP_200_OK) + + +class OccurrenceReportBulkImportTaskViewSet( + viewsets.GenericViewSet, + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.UpdateModelMixin, + mixins.ListModelMixin, +): + queryset = OccurrenceReportBulkImportTask.objects.all() + permission_classes = [OccurrenceReportBulkImportPermission] + serializer_class = OccurrenceReportBulkImportTaskSerializer + filter_backends = [filters.DjangoFilterBackend] + filterset_fields = ["processing_status"] + + def perform_create(self, serializer): + serializer.save(email_user=self.request.user.id) From 76bcb631f4a40643880f95367fa00e9f9600bc9a Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:38:38 +0800 Subject: [PATCH 12/47] Add basic django admin for OccurrenceReportBulkImportTask. --- boranga/components/occurrence/admin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/boranga/components/occurrence/admin.py b/boranga/components/occurrence/admin.py index 23ebc409..ede028a9 100644 --- a/boranga/components/occurrence/admin.py +++ b/boranga/components/occurrence/admin.py @@ -16,6 +16,7 @@ LocationAccuracy, ObservationMethod, OccurrenceGeometry, + OccurrenceReportBulkImportTask, OccurrenceReportGeometry, OccurrenceSite, OccurrenceTenure, @@ -466,6 +467,12 @@ class CountedSubjectAdmin(ArchivableModelAdminMixin, DeleteProtectedModelAdmin): list_display = ["name"] +class OccurrenceReportBulkImportTaskAdmin(DeleteProtectedModelAdmin): + list_display = ["id", "datetime_queued", "processing_status"] + list_filter = ["processing_status", "datetime_completed"] + readonly_fields = ["datetime_queued"] + + # Each of the following models will be available to Django Admin. admin.site.register(LandForm, LandFormAdmin) admin.site.register(RockType, RockTypeAdmin) @@ -494,3 +501,4 @@ class CountedSubjectAdmin(ArchivableModelAdminMixin, DeleteProtectedModelAdmin): admin.site.register(LocationAccuracy, LocationAccuracyAdmin) admin.site.register(WildStatus, WildStatusAdmin) admin.site.register(OccurrenceSite) +admin.site.register(OccurrenceReportBulkImportTask, OccurrenceReportBulkImportTaskAdmin) From 938aa916948076491fad2443cba5376fbcc2ecf9 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:39:33 +0800 Subject: [PATCH 13/47] Add bulk_import_task foreign key to OCR model (so we can keep track of those that came from a bulk import task). --- boranga/components/occurrence/models.py | 8 +++++ ...urrencereport_bulk_import_task_and_more.py | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 boranga/migrations/0429_occurrencereport_bulk_import_task_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 914d359c..8f4df8ed 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -290,6 +290,14 @@ class OccurrenceReport(SubmitterInformationModelMixin, RevisionedMixin): internal_application = models.BooleanField(default=False) site = models.TextField(null=True, blank=True) + bulk_import_task = models.ForeignKey( + "OccurrenceReportBulkImportTask", + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="occurrence_reports", + ) + class Meta: app_label = "boranga" ordering = ["-id"] diff --git a/boranga/migrations/0429_occurrencereport_bulk_import_task_and_more.py b/boranga/migrations/0429_occurrencereport_bulk_import_task_and_more.py new file mode 100644 index 00000000..cd8329c4 --- /dev/null +++ b/boranga/migrations/0429_occurrencereport_bulk_import_task_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 5.0.8 on 2024-08-15 08:36 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0428_occurrencereportbulkimporttask_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="occurrencereport", + name="bulk_import_task", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="occurrence_reports", + to="boranga.occurrencereportbulkimporttask", + ), + ), + migrations.AlterField( + model_name="occurrencereportbulkimporttask", + name="rows", + field=models.IntegerField(editable=False, null=True), + ), + ] From 32b061633edf64e0f54427d7b76c0cde90107163 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:39:56 +0800 Subject: [PATCH 14/47] Add occurrence_report_bulk_imports end point. --- boranga/frontend/boranga/src/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/boranga/frontend/boranga/src/api.js b/boranga/frontend/boranga/src/api.js index edb076cf..ed0ce60b 100644 --- a/boranga/frontend/boranga/src/api.js +++ b/boranga/frontend/boranga/src/api.js @@ -24,6 +24,7 @@ module.exports = { filtered_organisations: '/api/filtered_organisations', help_text_entries: "/api/help_text_entries", marine_treeview: "/api/marine_treeview", + occurrence_report_bulk_imports: "/api/occurrence_report_bulk_imports", ocr_external_referee_invites: "/api/ocr_external_referee_invites", ocr_referrals: "/api/ocr_referrals.json", organisation_access_group_members: '/api/organisation_access_group_members', From 76ea8faff905a77f6c01d0717de73c58f058b173 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 15 Aug 2024 16:40:28 +0800 Subject: [PATCH 15/47] Register occurrence_report_bulk_imports route. --- boranga/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/boranga/urls.py b/boranga/urls.py index 0a4e3132..1096ce6c 100755 --- a/boranga/urls.py +++ b/boranga/urls.py @@ -150,6 +150,11 @@ def trigger_error(request): occurrence_api.OccurrenceReportPaginatedViewSet, "occurrence_report_paginated", ) +router.register( + r"occurrence_report_bulk_imports", + occurrence_api.OccurrenceReportBulkImportTaskViewSet, + "occurrence_report_bulk_imports", +) router.register(r"observer_detail", occurrence_api.ObserverDetailViewSet) router.register(r"contact_detail", occurrence_api.ContactDetailViewSet) router.register(r"occurrence_sites", occurrence_api.OccurrenceSiteViewSet) From 3fc6f024edbcb9c5037714af775d1f39acfd71b2 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Fri, 16 Aug 2024 16:15:52 +0800 Subject: [PATCH 16/47] Work on bulk import facility. --- boranga/components/occurrence/api.py | 9 + boranga/components/occurrence/models.py | 377 +++++++++++++----- boranga/components/occurrence/serializers.py | 8 +- boranga/frontend/boranga/src/api.js | 2 +- .../internal/occurrence/bulk_import.vue | 222 +++++++++-- .../occurrence/bulk_import_schema.vue | 260 ++++++++++++ .../src/components/internal/routes/index.js | 6 + .../ocr_pre_process_bulk_import_tasks.py | 43 ++ .../commands/ocr_process_bulk_import_queue.py | 78 ++++ ...0_occurrencereport_import_hash_and_more.py | 66 +++ boranga/settings.py | 4 + boranga/static/boranga/css/base.css | 14 - .../webtemplate_dbca/includes/staff_menu.html | 5 +- python-cron | 1 + 14 files changed, 945 insertions(+), 150 deletions(-) create mode 100644 boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue create mode 100644 boranga/management/commands/ocr_pre_process_bulk_import_tasks.py create mode 100644 boranga/management/commands/ocr_process_bulk_import_queue.py create mode 100644 boranga/migrations/0430_occurrencereport_import_hash_and_more.py diff --git a/boranga/components/occurrence/api.py b/boranga/components/occurrence/api.py index 3060bcba..ee651e4a 100644 --- a/boranga/components/occurrence/api.py +++ b/boranga/components/occurrence/api.py @@ -22,6 +22,7 @@ from rest_framework.decorators import action as detail_route from rest_framework.decorators import action as list_route from rest_framework.decorators import renderer_classes +from rest_framework.pagination import LimitOffsetPagination from rest_framework.permissions import AllowAny from rest_framework.renderers import JSONRenderer from rest_framework.response import Response @@ -6248,6 +6249,14 @@ class OccurrenceReportBulkImportTaskViewSet( serializer_class = OccurrenceReportBulkImportTaskSerializer filter_backends = [filters.DjangoFilterBackend] filterset_fields = ["processing_status"] + pagination_class = LimitOffsetPagination def perform_create(self, serializer): serializer.save(email_user=self.request.user.id) + + @detail_route(methods=["patch"], detail=True) + def retry(self, request, *args, **kwargs): + instance = self.get_object() + instance.retry() + instance.save() + return Response(status=status.HTTP_200_OK) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 8f4df8ed..64b9684e 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -1,9 +1,12 @@ +import hashlib import json import logging import os from abc import abstractmethod from datetime import datetime +from decimal import Decimal +import dateutil import openpyxl import pyproj import reversion @@ -25,6 +28,7 @@ from ledger_api_client.ledger_models import EmailUserRO as EmailUser from ledger_api_client.managed_models import SystemGroup from multiselectfield import MultiSelectField +from openpyxl.worksheet.datavalidation import DataValidation from boranga import exceptions from boranga.components.conservation_status.models import ProposalAmendmentReason @@ -290,6 +294,7 @@ class OccurrenceReport(SubmitterInformationModelMixin, RevisionedMixin): internal_application = models.BooleanField(default=False) site = models.TextField(null=True, blank=True) + # If this OCR was created as part of a bulk import task, this field will be populated bulk_import_task = models.ForeignKey( "OccurrenceReportBulkImportTask", on_delete=models.PROTECT, @@ -297,6 +302,8 @@ class OccurrenceReport(SubmitterInformationModelMixin, RevisionedMixin): blank=True, related_name="occurrence_reports", ) + # A hash of the import row data to allow for duplicate detection + import_hash = models.CharField(max_length=64, null=True, blank=True) class Meta: app_label = "boranga" @@ -5157,87 +5164,6 @@ def save(self, *args, **kwargs): super().save(*args, **kwargs) -# Occurrence Report Document -reversion.register(OccurrenceReportDocument) - -# Occurrence Report Threat -reversion.register(OCRConservationThreat) - -# Occurrence Report Observer Detail -reversion.register(OCRObserverDetail) - -reversion.register(OCRHabitatComposition) -reversion.register(OCRHabitatCondition) -reversion.register(OCRVegetationStructure) -reversion.register(OCRFireHistory) -reversion.register(OCRAssociatedSpecies) -reversion.register(OCRObservationDetail) -reversion.register(OCRPlantCount) -reversion.register(OCRAnimalObservation) -reversion.register(OCRIdentification) - -# Occurrence Report -reversion.register( - OccurrenceReport, - follow=[ - "species", - "community", - "habitat_composition", - "habitat_condition", - "vegetation_structure", - "fire_history", - "associated_species", - "observation_detail", - "plant_count", - "animal_observation", - "identification", - ], -) - -# Occurrence Document -reversion.register(OccurrenceDocument) - -# Occurrence Threat -reversion.register(OCCConservationThreat) - -# Occurrence Contact Detail -reversion.register(OCCContactDetail) - -# Occurrence Site -reversion.register(OccurrenceSite) - -# Occurrence Tenure -reversion.register(OccurrenceTenure) - -reversion.register(OCCHabitatComposition) -reversion.register(OCCHabitatCondition) -reversion.register(OCCVegetationStructure) -reversion.register(OCCFireHistory) -reversion.register(OCCAssociatedSpecies) -reversion.register(OCCObservationDetail) -reversion.register(OCCPlantCount) -reversion.register(OCCAnimalObservation) -reversion.register(OCCIdentification) - -# Occurrence -reversion.register( - Occurrence, - follow=[ - "species", - "community", - "habitat_composition", - "habitat_condition", - "vegetation_structure", - "fire_history", - "associated_species", - "observation_detail", - "plant_count", - "animal_observation", - "identification", - ], -) - - def validate_bulk_import_file_extension(value): ext = os.path.splitext(value.name)[1] valid_extensions = [".xlsx"] @@ -5258,6 +5184,9 @@ class OccurrenceReportBulkImportTask(models.Model): storage=private_storage, validators=[validate_bulk_import_file_extension], ) + # A hash of the file to allow for duplicate detection + file_hash = models.CharField(max_length=64, null=True, blank=True) + rows = models.IntegerField(null=True, editable=False) rows_processed = models.IntegerField(default=0) @@ -5294,6 +5223,11 @@ class Meta: verbose_name = "Occurrence Report Bulk Import Task" verbose_name_plural = "Occurrence Report Bulk Import Tasks" + def save(self, *args, **kwargs): + if self._file: + self.file_hash = hashlib.sha256(self._file.read()).hexdigest() + super().save(*args, **kwargs) + @property def file_name(self): return os.path.basename(self._file.name) @@ -5301,19 +5235,40 @@ def file_name(self): @property def percentage_complete(self): if self.rows: - return round((self.rows_processed / self.rows) * 100) + return round((self.rows_processed / self.rows) * 100, 2) return 0 @property def total_time_taken(self): if self.datetime_started and self.datetime_completed: - return self.datetime_completed - self.datetime_started + delta = self.datetime_completed - self.datetime_started + return delta.seconds + return None + + @property + def total_time_taken_minues(self): + if self.total_time_taken: + return round(self.total_time_taken / 60, 2) + return None + + @property + def total_time_taken_human_readable(self): + if self.total_time_taken == 0: + return "Less than a second" + if not self.total_time_taken: + return None + + if self.total_time_taken < 60: + return f"{self.total_time_taken} seconds" + if self.total_time_taken: + return f"{self.total_time_taken_minues} minutes" return None @property def time_taken_per_row(self): if self.datetime_started and self.datetime_completed: - return self.total_time_taken() / self.rows_processed + value = self.total_time_taken / self.rows_processed + return round(value, 4) return None @property @@ -5333,27 +5288,49 @@ def average_time_taken_per_row(cls): task_count = cls.objects.filter( datetime_completed__isnull=False, rows_processed__gt=0 ).count() + if task_count == 0: + return None + total_time_taken = 0 for task in cls.objects.filter( datetime_completed__isnull=False, rows_processed__gt=0 ): - total_time_taken += task.average_time_taken_per_row() + total_time_taken += task.time_taken_per_row - return total_time_taken / task_count if task_count > 0 else None + return total_time_taken / task_count @property - def estimated_processing_time(self): + def estimated_processing_time_seconds(self): average_time_taken_per_row = ( OccurrenceReportBulkImportTask.average_time_taken_per_row() ) - if not average_time_taken_per_row: - return "No processing data available to estimate time" - if self.rows and self.datetime_queued: - return (self.rows - self.rows_processed) * average_time_taken_per_row + + if self.rows and self.datetime_queued and average_time_taken_per_row: + precisely = (self.rows - self.rows_processed) * average_time_taken_per_row + return round(precisely) + return None - @transaction.atomic - def pre_process(self): + @property + def estimated_processing_time_minutes(self): + seconds = self.estimated_processing_time_seconds + if seconds: + return round(seconds / 60) + return None + + @property + def estimated_processing_time_human_readable(self): + minutes = self.estimated_processing_time_minutes + + if not minutes: + return "No processing data available to estimate time" + + if minutes == 0: + return "Less than a minute" + + return f"~{minutes} minutes" + + def count_rows(self): logger.info(f"Pre-queue processing bulk import task {self.id}") try: workbook = openpyxl.load_workbook(self._file) @@ -5369,6 +5346,7 @@ def pre_process(self): sheet = workbook.active self.rows = sheet.max_row - 1 + logger.debug(f"Found {self.rows} rows in bulk import file {self._file.name}") self.save() def process(self): @@ -5393,7 +5371,7 @@ def process(self): # Open the file logger.info(f"Opening bulk import file {self._file.name}") try: - workbook = openpyxl.load_workbook(self._file) + workbook = openpyxl.load_workbook(self._file, read_only=True) except Exception as e: logger.error(f"Error opening bulk import file {self._file.name}: {e}") self.processing_status = ( @@ -5411,14 +5389,20 @@ def process(self): headers = [cell.value for cell in sheet[1]] # Get the rows - rows = list(sheet.iter_rows(min_row=2, values_only=True)) + rows = list(sheet.iter_rows(min_row=2, max_row=self.rows + 1, values_only=True)) # Process the rows for i, row in enumerate(rows): self.rows_processed = i + 1 + if self.rows_processed > self.rows: + logger.warning( + f"Bulk import task {self.id} tried to process row {i + 1} " + "which is greater than the total number of rows" + ) + break + self.save() - logger.info(f"Processing row {i + 1}") try: self.process_row(row, headers) except Exception as e: @@ -5436,11 +5420,208 @@ def process(self): self.processing_status = ( OccurrenceReportBulkImportTask.PROCESSING_STATUS_COMPLETED ) + self.datetime_completed = timezone.now() self.save() - logger.info(f"Processing bulk import task {self.id}") return def process_row(self, row, headers): - logger.info(f"Processing row {row}") + row_hash = hashlib.sha256(str(row).encode()).hexdigest() + logger.info(f"Row hash: {row_hash}") return + + def retry(self): + self.processing_status = self.PROCESSING_STATUS_QUEUED + self.datetime_started = None + self.datetime_completed = None + self.datetime_error = None + self.error_row = None + self.error_message = None + self.save() + + +class OccurrenceReportBulkImportSchema(models.Model): + name = models.CharField(max_length=255, blank=False, null=False) + version = models.IntegerField(default=1) + group_type = models.ForeignKey( + GroupType, on_delete=models.PROTECT, null=True, blank=True + ) + + class Meta: + app_label = "boranga" + verbose_name = "Occurrence Report Bulk Import Schema" + verbose_name_plural = "Occurrence Report Bulk Import Schemas" + + def __str__(self): + return f"{self.name} (Version: {self.version})" + + +class OccurrenceReportBulkImportSchemaColumn(models.Model): + schema = models.ForeignKey( + OccurrenceReportBulkImportSchema, + related_name="columns", + on_delete=models.CASCADE, + ) + import_content_type = models.ForeignKey( + ct_models.ContentType, + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="import_columns", + ) + import_field_name = models.CharField(max_length=50, blank=False, null=False) + + column_header_name = models.CharField(max_length=50, blank=False, null=False) + data_validation_type = models.CharField( + max_length=20, + choices=[(x, x) for x in DataValidation.type.values], + default="string", + ) + required = models.BooleanField(default=False) + default_value = models.CharField(max_length=255, blank=True, null=True) + + max_length = models.IntegerField(null=True, blank=True) + min_value = models.IntegerField(null=True, blank=True) + max_value = models.IntegerField(null=True, blank=True) + + list_lookup_class = models.ForeignKey( + ct_models.ContentType, + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="list_lookup_columns", + ) + list_lookup_field = models.CharField(max_length=50, blank=True, null=True) + + class Meta: + app_label = "boranga" + verbose_name = "Occurrence Report Bulk Import Schema Column" + verbose_name_plural = "Occurrence Report Bulk Import Schema Columns" + + def __str__(self): + return f"{self.name} ({self.schema.name})" + + def validate(self, value): + if self.data_validation_type == "whole": + if not isinstance(value, int): + raise ValidationError( + f"Default value for {self.column_header_name} must be an integer" + ) + + if self.min_value and value < self.min_value: + raise ValidationError( + f"Default value for {self.column_header_name} is too low" + ) + + if self.max_value and value > self.max_value: + raise ValidationError( + f"Default value for {self.column_header_name} is too high" + ) + if self.data_validation_type == "decimal": + try: + value = Decimal(value) + except Exception: + raise ValidationError( + f"Default value for {self.column_header_name} must be a decimal" + ) + if self.data_validation_type == "date": + try: + value = dateutil.parser.parse(value) + except Exception: + raise ValidationError( + f"Default value for {self.column_header_name} must be a date" + ) + if self.data_validation_type == "time": + try: + value = dateutil.parser.parse(value) + except Exception: + raise ValidationError( + f"Default value for {self.column_header_name} must be a time" + ) + + if self.max_length: + if len(value) > self.max_length: + raise ValidationError( + f"Default value for {self.column_header_name} is too long" + ) + + +# Occurrence Report Document +reversion.register(OccurrenceReportDocument) + +# Occurrence Report Threat +reversion.register(OCRConservationThreat) + +# Occurrence Report Observer Detail +reversion.register(OCRObserverDetail) + +reversion.register(OCRHabitatComposition) +reversion.register(OCRHabitatCondition) +reversion.register(OCRVegetationStructure) +reversion.register(OCRFireHistory) +reversion.register(OCRAssociatedSpecies) +reversion.register(OCRObservationDetail) +reversion.register(OCRPlantCount) +reversion.register(OCRAnimalObservation) +reversion.register(OCRIdentification) + +# Occurrence Report +reversion.register( + OccurrenceReport, + follow=[ + "species", + "community", + "habitat_composition", + "habitat_condition", + "vegetation_structure", + "fire_history", + "associated_species", + "observation_detail", + "plant_count", + "animal_observation", + "identification", + ], +) + +# Occurrence Document +reversion.register(OccurrenceDocument) + +# Occurrence Threat +reversion.register(OCCConservationThreat) + +# Occurrence Contact Detail +reversion.register(OCCContactDetail) + +# Occurrence Site +reversion.register(OccurrenceSite) + +# Occurrence Tenure +reversion.register(OccurrenceTenure) + +reversion.register(OCCHabitatComposition) +reversion.register(OCCHabitatCondition) +reversion.register(OCCVegetationStructure) +reversion.register(OCCFireHistory) +reversion.register(OCCAssociatedSpecies) +reversion.register(OCCObservationDetail) +reversion.register(OCCPlantCount) +reversion.register(OCCAnimalObservation) +reversion.register(OCCIdentification) + +# Occurrence +reversion.register( + Occurrence, + follow=[ + "species", + "community", + "habitat_composition", + "habitat_condition", + "vegetation_structure", + "fire_history", + "associated_species", + "observation_detail", + "plant_count", + "animal_observation", + "identification", + ], +) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index ebff4227..206a9c3c 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -3861,9 +3861,11 @@ def get_last_updated_by(self, obj): class OccurrenceReportBulkImportTaskSerializer(serializers.ModelSerializer): - estimated_processing_time = serializers.CharField(read_only=True) + estimated_processing_time_human_readable = serializers.CharField(read_only=True) + total_time_taken_human_readable = serializers.CharField(read_only=True) file_size_megabytes = serializers.CharField(read_only=True) file_name = serializers.CharField(read_only=True) + percentage_complete = serializers.CharField(read_only=True) class Meta: model = OccurrenceReportBulkImportTask @@ -3880,5 +3882,7 @@ class Meta: "error_message", "processing_status", "email_user", - "estimated_processing_time", + "estimated_processing_time_human_readable", + "total_time_taken_human_readable", + "percentage_complete", ) diff --git a/boranga/frontend/boranga/src/api.js b/boranga/frontend/boranga/src/api.js index ed0ce60b..1bdc8558 100644 --- a/boranga/frontend/boranga/src/api.js +++ b/boranga/frontend/boranga/src/api.js @@ -24,7 +24,7 @@ module.exports = { filtered_organisations: '/api/filtered_organisations', help_text_entries: "/api/help_text_entries", marine_treeview: "/api/marine_treeview", - occurrence_report_bulk_imports: "/api/occurrence_report_bulk_imports", + occurrence_report_bulk_imports: "/api/occurrence_report_bulk_imports/", ocr_external_referee_invites: "/api/ocr_external_referee_invites", ocr_referrals: "/api/ocr_referrals.json", organisation_access_group_members: '/api/organisation_access_group_members', diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue index 59b7782d..3273a7ce 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue @@ -8,7 +8,8 @@
- Some information about the import process. Including a link to an example template .xlsx file? + Some information about the import process. Including a link to an example template + .xlsx file?
@@ -18,11 +19,12 @@
-
@@ -36,6 +38,7 @@ + @@ -43,15 +46,14 @@ - - - - - + + + + + +
Datetime Queued File Name File Size Row Count
R1C2R1C3R1C3 -
- 10 minutes -
-
{{ new Date(queuedImport.datetime_queued).toLocaleString() }}{{ + queuedImport.file_name }}{{ queuedImport.file_size_megabytes }} MB{{ queuedImport.rows ? queuedImport.rows : 'Not + Counted' }}{{ queuedImport.estimated_processing_time_human_readable }}
@@ -71,21 +73,26 @@ + - - - - - - + + + + @@ -96,7 +103,68 @@ -
+
+
+
+
Failed Bulk + Imports
+
+
Datetime Started File Name File SizeRow Count Progress
R1C2R1C3R1C3 +
{{ new Date(currentlyRunningImport.datetime_started).toLocaleString() }} + {{ currentlyRunningImport.file_name }}{{ currentlyRunningImport.file_size_megabytes }} MB
-
27% +
{{ + currentlyRunningImport.percentage_complete }}%
+ + + + + + + + + + + + + + + + + + +
Datetime StartedFile NameFile SizeRecords ImportedActions
{{ new Date(failedImport.datetime_started).toLocaleString() }}{{ + failedImport.file_name }}{{ failedImport.file_size_megabytes }} MB{{ failedImport.rows ? failedImport.rows : + 'Not Counted' }} + + +
+
+

load more

+
+ + + + +
Completed Bulk @@ -105,18 +173,21 @@ - + - + - - - - - + + + + +
File NameDatetime Completed File Size Records ImportedDate CompletedTotal Time Taken
R1C2R1C3R1C3R1C3
{{ new Date(completedImport.datetime_completed).toLocaleString() }} + {{ + completedImport.file_name }}{{ completedImport.rows_processed + }}{{ completedImport.total_time_taken_human_readable }}
@@ -133,14 +204,20 @@ diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue new file mode 100644 index 00000000..64ee6797 --- /dev/null +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue @@ -0,0 +1,260 @@ + + + + + diff --git a/boranga/frontend/boranga/src/components/internal/routes/index.js b/boranga/frontend/boranga/src/components/internal/routes/index.js index bb30094d..ad6a7dae 100755 --- a/boranga/frontend/boranga/src/components/internal/routes/index.js +++ b/boranga/frontend/boranga/src/components/internal/routes/index.js @@ -11,6 +11,7 @@ import Occurrence from '../occurrence/occurrence.vue' import OccurrenceReport from '../occurrence/occurrence_report.vue' import OccurrenceReportReferral from '../occurrence/referral.vue' import BulkImport from '../occurrence/bulk_import.vue' +import BulkImportSchema from '../occurrence/bulk_import_schema.vue' export default { @@ -64,6 +65,11 @@ export default }, }, children: [ + { + path: 'bulk_import_schema/', + name: "occurrence-report-bulk-import-schema", + component: BulkImportSchema + }, { path: 'bulk_import/', name: "occurrence-report-bulk-import", diff --git a/boranga/management/commands/ocr_pre_process_bulk_import_tasks.py b/boranga/management/commands/ocr_pre_process_bulk_import_tasks.py new file mode 100644 index 00000000..c885dca7 --- /dev/null +++ b/boranga/management/commands/ocr_pre_process_bulk_import_tasks.py @@ -0,0 +1,43 @@ +import logging + +from django.core.management.base import BaseCommand + +from boranga.components.occurrence.models import OccurrenceReportBulkImportTask + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Pre process the OCR bulk import tasks" + + def handle(self, *args, **options): + logger.info(f"Running command {__name__}") + + # Check if there are already any tasks running and return if so + if OccurrenceReportBulkImportTask.objects.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED + ).exists(): + logger.info("There is already a task running, returning") + return + + # Get the next task to process + task = ( + OccurrenceReportBulkImportTask.objects.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED, + _file__isnull=False, + rows__isnull=True, + ) + .order_by("datetime_queued") + .first() + ) + + if task is None: + logger.info("No tasks to process, returning") + return + + # Process the task + task.count_rows() + + logger.info(f"OCR Bulk Import Task {task.id} has {task.rows} rows.") + + return diff --git a/boranga/management/commands/ocr_process_bulk_import_queue.py b/boranga/management/commands/ocr_process_bulk_import_queue.py new file mode 100644 index 00000000..ea434fa8 --- /dev/null +++ b/boranga/management/commands/ocr_process_bulk_import_queue.py @@ -0,0 +1,78 @@ +import logging + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.utils import timezone + +from boranga.components.occurrence.models import OccurrenceReportBulkImportTask + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Process the OCR bulk import queue" + + def handle(self, *args, **options): + logger.info(f"Running command {__name__}") + + # Check if there are any tasks that have been processing for too long + qs = OccurrenceReportBulkImportTask.objects.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED, + datetime_started__lt=timezone.now() + - timezone.timedelta(seconds=settings.OCR_BULK_IMPORT_TASK_TIMEOUT_SECONDS), + ) + if qs.exists(): + for task in qs: + logger.info( + f"Task {task.id} has been processing for too long. Adding back to the queue" + ) + task.processing_status = ( + OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED + ) + task.rows_processed = 0 + task.save() + + # Check if there are already any tasks running and return if so + if OccurrenceReportBulkImportTask.objects.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED, + ).exists(): + logger.info("There is already a task running, returning") + return + + # Get the next task to process + task = ( + OccurrenceReportBulkImportTask.objects.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED, + _file__isnull=False, + ) + .order_by("datetime_queued") + .first() + ) + + if task is None: + logger.info("No tasks to process, returning") + return + + try: + # Process the task + task.process() + except KeyboardInterrupt: + logger.info(f"OCR Bulk Import Task {task.id} was interrupted") + task.processing_status = ( + OccurrenceReportBulkImportTask.PROCESSING_STATUS_FAILED + ) + task.error_message = "KeyboardInterrupt" + task.save() + return + except Exception as e: + logger.error(f"Error processing OCR Bulk Import Task {task.id}: {e}") + task.processing_status = ( + OccurrenceReportBulkImportTask.PROCESSING_STATUS_FAILED + ) + task.error_message = str(e) + task.save() + return + + logger.info(f"OCR Bulk Import Task {task.id} completed") + + return diff --git a/boranga/migrations/0430_occurrencereport_import_hash_and_more.py b/boranga/migrations/0430_occurrencereport_import_hash_and_more.py new file mode 100644 index 00000000..af3f612e --- /dev/null +++ b/boranga/migrations/0430_occurrencereport_import_hash_and_more.py @@ -0,0 +1,66 @@ +# Generated by Django 5.0.8 on 2024-08-16 06:14 + +import boranga.components.conservation_status.models +import boranga.components.meetings.models +import boranga.components.occurrence.models +import boranga.components.species_and_communities.models +import boranga.components.users.models +import django.core.files.storage +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boranga', '0429_occurrencereport_bulk_import_task_and_more'), + ('contenttypes', '0002_remove_content_type_name'), + ] + + operations = [ + migrations.AddField( + model_name='occurrencereport', + name='import_hash', + field=models.CharField(blank=True, max_length=64, null=True), + ), + migrations.AddField( + model_name='occurrencereportbulkimporttask', + name='file_hash', + field=models.CharField(blank=True, max_length=64, null=True), + ), + migrations.CreateModel( + name='OccurrenceReportBulkImportSchema', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('version', models.IntegerField(default=1)), + ('group_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='boranga.grouptype')), + ], + options={ + 'verbose_name': 'Occurrence Report Bulk Import Schema', + 'verbose_name_plural': 'Occurrence Report Bulk Import Schemas', + }, + ), + migrations.CreateModel( + name='OccurrenceReportBulkImportSchemaColumn', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('import_field_name', models.CharField(max_length=50)), + ('column_header_name', models.CharField(max_length=50)), + ('data_validation_type', models.CharField(choices=[('textLength', 'textLength'), ('time', 'time'), ('list', 'list'), ('custom', 'custom'), (None, None), ('whole', 'whole'), ('decimal', 'decimal'), ('date', 'date')], default='string', max_length=20)), + ('required', models.BooleanField(default=False)), + ('default_value', models.CharField(blank=True, max_length=255, null=True)), + ('max_length', models.IntegerField(blank=True, null=True)), + ('min_value', models.IntegerField(blank=True, null=True)), + ('max_value', models.IntegerField(blank=True, null=True)), + ('list_lookup_field', models.CharField(blank=True, max_length=50, null=True)), + ('import_content_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='import_columns', to='contenttypes.contenttype')), + ('list_lookup_class', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='list_lookup_columns', to='contenttypes.contenttype')), + ('schema', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='columns', to='boranga.occurrencereportbulkimportschema')), + ], + options={ + 'verbose_name': 'Occurrence Report Bulk Import Schema Column', + 'verbose_name_plural': 'Occurrence Report Bulk Import Schema Columns', + }, + ), + ] diff --git a/boranga/settings.py b/boranga/settings.py index a3d1af8a..f98d440f 100755 --- a/boranga/settings.py +++ b/boranga/settings.py @@ -466,3 +466,7 @@ def show_toolbar(request): # (_save method of FileSystemStorage class) # As it causes a permission exception when using azure network drives FILE_UPLOAD_PERMISSIONS = None + +OCR_BULK_IMPORT_TASK_TIMEOUT_SECONDS = env( + "OCR_BULK_IMPORT_TASK_TIMEOUT_SECONDS", 60 * 5 +) # Default = 5 minutes diff --git a/boranga/static/boranga/css/base.css b/boranga/static/boranga/css/base.css index f7f3141c..4ae2181d 100755 --- a/boranga/static/boranga/css/base.css +++ b/boranga/static/boranga/css/base.css @@ -129,20 +129,6 @@ body { overflow-x: scroll; } -.modal-header { - background-color: #529b6b; - color: #fcb13f; -} - -.modal-body { - background-color: #529b6b; - color: #fff; -} - -.modal-footer { - background-color: #529b6b; -} - .popover { max-width: 100%; } diff --git a/boranga/templates/webtemplate_dbca/includes/staff_menu.html b/boranga/templates/webtemplate_dbca/includes/staff_menu.html index 07b1e69b..8d808e22 100644 --- a/boranga/templates/webtemplate_dbca/includes/staff_menu.html +++ b/boranga/templates/webtemplate_dbca/includes/staff_menu.html @@ -2,4 +2,7 @@ {% is_django_admin as is_django_admin_user %} {% if is_django_admin_user %}
  • Admin
    • -{% endif %} \ No newline at end of file +{% endif %} +{% if is_django_admin_user or request.user.is_superuser %} +
  • OCR Bulk Import Schema
    • +{% endif %} diff --git a/python-cron b/python-cron index 3df1cba4..46b7d72a 100644 --- a/python-cron +++ b/python-cron @@ -8,3 +8,4 @@ # * * * * * Command */5 * * * * /app/venv/bin/python3 /app/manage.py runcrons >> /app/logs/cronjob.log 2>&1 +1 * * * * * /app/venv/bin/python3 /app/manage.py runcrons >> /app/logs/cronjob.log 2>&1 From ec3a7d42bc446a141be2babf5bb4e6f2e950b858 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 10:04:26 +0800 Subject: [PATCH 17/47] Fix total_time_taken to return milliseconds. Make OccurrenceReportBulkImportTask model archivable. --- boranga/components/occurrence/models.py | 13 +++++-- ...ereportbulkimporttask_archived_and_more.py | 36 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 boranga/migrations/0431_occurrencereportbulkimporttask_archived_and_more.py diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 64b9684e..21da26ee 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -5177,7 +5177,7 @@ def get_occurrence_report_bulk_import_path(instance, filename): return f"occurrence_report/bulk-imports/{timezone.now()}/{filename}" -class OccurrenceReportBulkImportTask(models.Model): +class OccurrenceReportBulkImportTask(ArchivableModel): _file = models.FileField( upload_to=get_occurrence_report_bulk_import_path, max_length=512, @@ -5240,6 +5240,13 @@ def percentage_complete(self): @property def total_time_taken(self): + if self.datetime_started and self.datetime_completed: + delta = self.datetime_completed - self.datetime_started + return delta.total_seconds() + return None + + @property + def total_time_taken_seconds(self): if self.datetime_started and self.datetime_completed: delta = self.datetime_completed - self.datetime_started return delta.seconds @@ -5253,7 +5260,7 @@ def total_time_taken_minues(self): @property def total_time_taken_human_readable(self): - if self.total_time_taken == 0: + if self.total_time_taken < 1: return "Less than a second" if not self.total_time_taken: return None @@ -5268,7 +5275,7 @@ def total_time_taken_human_readable(self): def time_taken_per_row(self): if self.datetime_started and self.datetime_completed: value = self.total_time_taken / self.rows_processed - return round(value, 4) + return round(value, 6) return None @property diff --git a/boranga/migrations/0431_occurrencereportbulkimporttask_archived_and_more.py b/boranga/migrations/0431_occurrencereportbulkimporttask_archived_and_more.py new file mode 100644 index 00000000..c12bcbeb --- /dev/null +++ b/boranga/migrations/0431_occurrencereportbulkimporttask_archived_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0.8 on 2024-08-19 02:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0430_occurrencereport_import_hash_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="occurrencereportbulkimporttask", + name="archived", + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="data_validation_type", + field=models.CharField( + choices=[ + ("textLength", "textLength"), + ("list", "list"), + ("time", "time"), + ("date", "date"), + ("whole", "whole"), + (None, None), + ("decimal", "decimal"), + ("custom", "custom"), + ], + default="string", + max_length=20, + ), + ), + ] From b639b5f00f3ecac51fc3773aa0d0806ec6138370 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 15:54:28 +0800 Subject: [PATCH 18/47] Add end points: occurrence_report_bulk_import_schemas, occurrence_report_bulk_import_schemas_by_group_type --- boranga/frontend/boranga/src/api.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/boranga/frontend/boranga/src/api.js b/boranga/frontend/boranga/src/api.js index 1bdc8558..06ef6513 100644 --- a/boranga/frontend/boranga/src/api.js +++ b/boranga/frontend/boranga/src/api.js @@ -25,6 +25,8 @@ module.exports = { help_text_entries: "/api/help_text_entries", marine_treeview: "/api/marine_treeview", occurrence_report_bulk_imports: "/api/occurrence_report_bulk_imports/", + occurrence_report_bulk_import_schemas: "/api/occurrence_report_bulk_import_schemas/", + occurrence_report_bulk_import_schemas_by_group_type: "/api/occurrence_report_bulk_import_schemas/get_schema_list_by_group_type/", ocr_external_referee_invites: "/api/ocr_external_referee_invites", ocr_referrals: "/api/ocr_referrals.json", organisation_access_group_members: '/api/organisation_access_group_members', From 96de31e45cf78db97f2d62a62004de7f860e00c9 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 15:55:05 +0800 Subject: [PATCH 19/47] Add OccurrenceReportBulkImportSchemaColumnSerializer, OccurrenceReportBulkImportSchemaSerializer. Add create method to OccurrenceReportBulkImportTaskSerializer. --- boranga/components/occurrence/serializers.py | 43 ++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index 206a9c3c..74083de3 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -1,3 +1,4 @@ +import hashlib import logging from django.db import models @@ -36,6 +37,8 @@ OccurrenceReportAmendmentRequest, OccurrenceReportAmendmentRequestDocument, OccurrenceReportApprovalDetails, + OccurrenceReportBulkImportSchema, + OccurrenceReportBulkImportSchemaColumn, OccurrenceReportBulkImportTask, OccurrenceReportDeclinedDetails, OccurrenceReportDocument, @@ -3886,3 +3889,43 @@ class Meta: "total_time_taken_human_readable", "percentage_complete", ) + + def create(self, validated_data): + file_hash = hashlib.sha256(validated_data["_file"].read()).hexdigest() + qs = OccurrenceReportBulkImportTask.objects.filter(file_hash=file_hash) + if qs.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED + ).exists(): + raise serializers.ValidationError( + "An import task with exactly same file contents has already been queued." + ) + if qs.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED + ).exists(): + raise serializers.ValidationError( + "An import task with exactly same file contents is already in progress." + ) + if qs.filter( + processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_COMPLETED + ).exists(): + raise serializers.ValidationError( + "An import task with exactly same file contents has already been completed." + ) + return super().create(validated_data) + + +class OccurrenceReportBulkImportSchemaColumnSerializer(serializers.ModelSerializer): + + class Meta: + model = OccurrenceReportBulkImportSchemaColumn + fields = "__all__" + read_only_fields = ("id",) + + +class OccurrenceReportBulkImportSchemaSerializer(serializers.ModelSerializer): + columns = OccurrenceReportBulkImportSchemaColumnSerializer(many=True) + + class Meta: + model = OccurrenceReportBulkImportSchema + fields = "__all__" + read_only_fields = ("id",) From 8562e091da84b8adf21a5e6eb409e2e0022c59b1 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 15:55:40 +0800 Subject: [PATCH 20/47] Add basic admin for OccurrenceReportBulkImportSchema. --- boranga/components/occurrence/admin.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/boranga/components/occurrence/admin.py b/boranga/components/occurrence/admin.py index ede028a9..12cb685a 100644 --- a/boranga/components/occurrence/admin.py +++ b/boranga/components/occurrence/admin.py @@ -16,6 +16,8 @@ LocationAccuracy, ObservationMethod, OccurrenceGeometry, + OccurrenceReportBulkImportSchema, + OccurrenceReportBulkImportSchemaColumn, OccurrenceReportBulkImportTask, OccurrenceReportGeometry, OccurrenceSite, @@ -473,6 +475,19 @@ class OccurrenceReportBulkImportTaskAdmin(DeleteProtectedModelAdmin): readonly_fields = ["datetime_queued"] +class OccurrenceReportBulkImportSchemaColumnInline(admin.StackedInline): + model = OccurrenceReportBulkImportSchemaColumn + extra = 0 + + +class OccurrenceReportBulkImportSchemaAdmin(DeleteProtectedModelAdmin): + list_display = ["group_type", "version", "datetime_created", "datetime_updated"] + readonly_fields = ["datetime_created", "datetime_updated"] + list_filter = ["group_type"] + inlines = [OccurrenceReportBulkImportSchemaColumnInline] + ordering = ["group_type", "version"] + + # Each of the following models will be available to Django Admin. admin.site.register(LandForm, LandFormAdmin) admin.site.register(RockType, RockTypeAdmin) @@ -502,3 +517,6 @@ class OccurrenceReportBulkImportTaskAdmin(DeleteProtectedModelAdmin): admin.site.register(WildStatus, WildStatusAdmin) admin.site.register(OccurrenceSite) admin.site.register(OccurrenceReportBulkImportTask, OccurrenceReportBulkImportTaskAdmin) +admin.site.register( + OccurrenceReportBulkImportSchema, OccurrenceReportBulkImportSchemaAdmin +) From 19f0d0ef2d1ca5161542c8ef214396a481a4a665 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 16:52:51 +0800 Subject: [PATCH 21/47] Register route for OccurrenceReportBulkImportSchemaViewSet. --- boranga/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/boranga/urls.py b/boranga/urls.py index 1096ce6c..7ddcd9be 100755 --- a/boranga/urls.py +++ b/boranga/urls.py @@ -155,6 +155,11 @@ def trigger_error(request): occurrence_api.OccurrenceReportBulkImportTaskViewSet, "occurrence_report_bulk_imports", ) +router.register( + r"occurrence_report_bulk_import_schemas", + occurrence_api.OccurrenceReportBulkImportSchemaViewSet, + "occurrence_report_bulk_import_schemas", +) router.register(r"observer_detail", occurrence_api.ObserverDetailViewSet) router.register(r"contact_detail", occurrence_api.ContactDetailViewSet) router.register(r"occurrence_sites", occurrence_api.OccurrenceSiteViewSet) From 2308588c251f7caf3c54612ee36668bc1bcbc7e2 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 16:53:40 +0800 Subject: [PATCH 22/47] Add OccurrenceReportBulkImportSchemaViewSet. Add revert method to OccurrenceReportBulkImportTaskViewSet. --- boranga/components/occurrence/api.py | 55 ++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/boranga/components/occurrence/api.py b/boranga/components/occurrence/api.py index ee651e4a..8497324c 100644 --- a/boranga/components/occurrence/api.py +++ b/boranga/components/occurrence/api.py @@ -67,6 +67,7 @@ OccurrenceReport, OccurrenceReportAmendmentRequest, OccurrenceReportAmendmentRequestDocument, + OccurrenceReportBulkImportSchema, OccurrenceReportBulkImportTask, OccurrenceReportDocument, OccurrenceReportGeometry, @@ -136,6 +137,7 @@ OccurrenceDocumentSerializer, OccurrenceLogEntrySerializer, OccurrenceReportAmendmentRequestSerializer, + OccurrenceReportBulkImportSchemaSerializer, OccurrenceReportBulkImportTaskSerializer, OccurrenceReportDocumentSerializer, OccurrenceReportLogEntrySerializer, @@ -6239,9 +6241,7 @@ def retract(self, request, *args, **kwargs): class OccurrenceReportBulkImportTaskViewSet( viewsets.GenericViewSet, - mixins.RetrieveModelMixin, mixins.CreateModelMixin, - mixins.UpdateModelMixin, mixins.ListModelMixin, ): queryset = OccurrenceReportBulkImportTask.objects.all() @@ -6258,5 +6258,54 @@ def perform_create(self, serializer): def retry(self, request, *args, **kwargs): instance = self.get_object() instance.retry() - instance.save() return Response(status=status.HTTP_200_OK) + + @detail_route(methods=["patch"], detail=True) + def revert(self, request, *args, **kwargs): + instance = self.get_object() + instance.revert() + return Response(status=status.HTTP_200_OK) + + +class OccurrenceReportBulkImportSchemaViewSet( + viewsets.GenericViewSet, + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.ListModelMixin, +): + queryset = OccurrenceReportBulkImportSchema.objects.all() + serializer_class = OccurrenceReportBulkImportSchemaSerializer + permission_classes = [OccurrenceReportBulkImportPermission] + + def get_queryset(self): + qs = self.queryset + if not (is_internal(self.request) or self.request.user.is_superuser): + qs = OccurrenceReportBulkImportSchema.objects.none() + return qs + + @list_route(methods=["get"], detail=False) + def get_schema_list_by_group_type(self, request, *args, **kwargs): + group_type = request.GET.get("group_type", None) + if not group_type: + raise serializers.ValidationError( + "Group Type is required to return correct list of values" + ) + + group_type = GroupType.objects.get(name=group_type) + + schema = OccurrenceReportBulkImportSchema.objects.filter(group_type=group_type) + serializer = OccurrenceReportBulkImportSchemaSerializer(schema, many=True) + return Response(serializer.data) + + @detail_route(methods=["get"], detail=True) + def preview_import_file(self, request, *args, **kwargs): + instance = self.get_object() + buffer = BytesIO() + workbook = instance.preview_import_file + workbook.save(buffer) + buffer.seek(0) + filename = f"bulk-import-schema-{instance.group_type.name}-version-{instance.version}-preview.xlsx" + response = HttpResponse(buffer.read(), content_type="application/vnd.ms-excel") + response["Content-Disposition"] = f"attachment; filename={filename}" + buffer.close() + return response From 622a6aba9ce50ef3d6525c334867cafd6083918e Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 16:54:10 +0800 Subject: [PATCH 23/47] Further configuration of bulk import component. --- .../internal/occurrence/bulk_import.vue | 211 ++++++++++++++---- 1 file changed, 166 insertions(+), 45 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue index 3273a7ce..7a76cb6a 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue @@ -15,18 +15,61 @@
    -
    - -
    -
    - - -
    +
    +
    + + +
    +
    + +
    + + + + + + + + + + + +
    {{ + column.xlsx_column_header_name }}
    {{ + column.xlsx_data_validation_type ? + column.xlsx_data_validation_type : 'None' }}
    +
    + +
    +
    + +
    + +
    +
      +
    • {{ error }}
      • +
    +
    +
    +
    +
    @@ -38,11 +81,14 @@ - - - - - + + + + + @@ -73,10 +119,13 @@
    Datetime QueuedFile NameFile SizeRow Count Time Estimate Datetime Queued + File Name + File Size Row Count Time + Estimate
    - - - - + + + + @@ -120,7 +169,8 @@ - + @@ -130,9 +180,11 @@ @@ -173,10 +225,15 @@
    Datetime StartedFile NameFile SizeProgress Datetime Started + File Name + File Size Progress +
    {{ new Date(failedImport.datetime_started).toLocaleString() }} {{ failedImport.file_name }} -
    - - - - + + + + + @@ -188,6 +245,12 @@ +
    Datetime CompletedFile SizeRecords ImportedTotal Time Taken Datetime + Completed File Name + Records Imported + Total + Time TakenActions
    {{ completedImport.rows_processed }} {{ completedImport.total_time_taken_human_readable }} + +
    @@ -210,7 +273,7 @@ export default { name: 'OccurrenceReportBulkImport', data() { return { - bulkImportButtonVisible: false, + form: null, queuedImports: null, currentlyRunningImports: null, failedImports: null, @@ -218,6 +281,9 @@ export default { timer: null, currentlyRunningTimer: null, selectedErrors: '', + schema_versions: [], + selected_schema_version: null, + importFileErrors: null } }, components: { @@ -229,13 +295,18 @@ export default { } }, methods: { + getSchemaVersionText(schema_version) { + return `Version: ${schema_version.version} (Created: ${new Date(schema_version.datetime_created).toLocaleDateString()} ${new Date(schema_version.datetime_created).toLocaleTimeString()}, Updated: ${new Date(schema_version.datetime_updated).toLocaleDateString()} ${new Date(schema_version.datetime_updated).toLocaleTimeString()})`; + }, + resetFileField() { + this.$nextTick(() => { + this.importFileErrors = null; + this.form.classList.remove('was-validated'); + }); + }, bulkImportFileSelected(event) { - console.log('Bulk Import File Selected'); - - if (!event.target.files.length) { - this.bulkImportButtonVisible = false; - return; - } + this.importFileErrors = null; + this.form.classList.remove('was-validated'); // If there is a file call the initial import file check api end point const file = event.target.files[0]; @@ -243,16 +314,32 @@ export default { formData.append('_file', file); this.$http.post(api_endpoints.occurrence_report_bulk_imports, formData).then((response) => { - if (response.status === 200) { - // If there is a file then make the bulk import button visible - this.bulkImportButtonVisible = true; + if (response.status >= 200 && response.status < 300) { console.log(response.body); + this.importFileErrors = null; + this.form.classList.remove('was-validated'); + this.$refs['bulk-import-file'].value = ''; + swal.fire({ + title: 'Bulk Import Added to Queue', + text: 'The bulk import of occurrence reports has been added to the queue for processing', + icon: 'success', + showCancelButton: false, + confirmButtonText: 'Ok', + customClass: { + confirmButton: 'btn btn-primary', + }, + }); + } else { + this.importFileErrors = response.body; + this.$refs['bulk-import-file'].setCustomValidity('Invalid field'); + this.form.classList.add('was-validated'); } }, (error) => { - console.log(error); + this.importFileErrors = error.body; + this.$refs['bulk-import-file'].setCustomValidity('Invalid field'); + this.form.classList.add('was-validated'); + console.log(error.body); }); - - // If there are any issues with the file then display the errors or warnings above }, confirmBeginBulkImport() { swal.fire({ @@ -272,11 +359,13 @@ export default { this.queueBulkImport(); } }); - console.log('Queue Bulk Import'); - // Call the bulk import api end point }, - queueBulkImport() { - // Call the api to add the import to the queue + fetchSchemas() { + this.$http.get(`${api_endpoints.occurrence_report_bulk_import_schemas_by_group_type}?group_type=${this.$route.query.group_type}`).then((response) => { + this.schema_versions = response.body; + }, (error) => { + console.log(error); + }); }, fetchQueuedImports() { this.$http.get(`${api_endpoints.occurrence_report_bulk_imports}?processing_status=queued`).then((response) => { @@ -319,6 +408,36 @@ export default { console.log(error); }); }, + revert(bulkImportTaskId) { + swal.fire({ + title: 'Revert and Archive', + text: 'Are you sure you want to revert this bulk import of occurrence reports and archive it?', + icon: 'question', + showCancelButton: true, + confirmButtonText: 'Revert and Archive', + cancelButtonText: 'Cancel', + customClass: { + confirmButton: 'btn btn-primary', + cancelButton: 'btn btn-secondary' + }, + reverseButtons: true + }).then((result) => { + if (result.isConfirmed) { + // Call the api to revert the bulk import task + this.$http.patch(`${api_endpoints.occurrence_report_bulk_imports}${bulkImportTaskId}/revert/`).then((response) => { + console.log(response); + // Remove the completed import from the completed imports list + this.completedImports = this.completedImports.filter((completedImport) => { + return completedImport.id !== bulkImportTaskId; + }); + this.fetchQueuedImports(); + }, (error) => { + console.log(error); + }); + } + }); + + }, fetchImports() { this.fetchQueuedImports(); this.fetchFailedImports(); @@ -326,10 +445,12 @@ export default { } }, created() { + this.fetchSchemas(); this.fetchImports(); this.fetchCurrentlyRunningImports(); }, mounted() { + this.form = document.getElementById('bulk-import-form'); this.timer = setInterval(() => { this.fetchImports() }, 5000) From 135a797cfe2411ef7ef6d0aff0498d4fd2c47fac Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 19 Aug 2024 16:54:32 +0800 Subject: [PATCH 24/47] Migration changes for bulk import models. --- ...kimportschema_datetime_created_and_more.py | 61 +++++++++++ ...tbulkimportschemacolumn_schema_and_more.py | 31 ++++++ ..._alter_communitydocument__file_and_more.py | 56 ++++++++++ ..._alter_communitydocument__file_and_more.py | 101 ++++++++++++++++++ ...reportbulkimportschema_options_and_more.py | 60 +++++++++++ ...ncereportbulkimporttask_schema_and_more.py | 63 +++++++++++ ..._alter_communitydocument__file_and_more.py | 52 +++++++++ 7 files changed, 424 insertions(+) create mode 100644 boranga/migrations/0432_occurrencereportbulkimportschema_datetime_created_and_more.py create mode 100644 boranga/migrations/0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more.py create mode 100644 boranga/migrations/0434_alter_communitydocument__file_and_more.py create mode 100644 boranga/migrations/0435_alter_communitydocument__file_and_more.py create mode 100644 boranga/migrations/0436_alter_occurrencereportbulkimportschema_options_and_more.py create mode 100644 boranga/migrations/0437_occurrencereportbulkimporttask_schema_and_more.py create mode 100644 boranga/migrations/0438_alter_communitydocument__file_and_more.py diff --git a/boranga/migrations/0432_occurrencereportbulkimportschema_datetime_created_and_more.py b/boranga/migrations/0432_occurrencereportbulkimportschema_datetime_created_and_more.py new file mode 100644 index 00000000..f147efce --- /dev/null +++ b/boranga/migrations/0432_occurrencereportbulkimportschema_datetime_created_and_more.py @@ -0,0 +1,61 @@ +# Generated by Django 5.0.8 on 2024-08-19 03:25 + +import datetime +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0431_occurrencereportbulkimporttask_archived_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="occurrencereportbulkimportschema", + name="datetime_created", + field=models.DateTimeField( + auto_now_add=True, default=django.utils.timezone.now + ), + preserve_default=False, + ), + migrations.AddField( + model_name="occurrencereportbulkimportschema", + name="datetime_updated", + field=models.DateTimeField(default=datetime.datetime.now), + ), + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="data_validation_type", + field=models.CharField( + choices=[ + ("whole", "whole"), + ("time", "time"), + ("decimal", "decimal"), + ("custom", "custom"), + ("list", "list"), + ("date", "date"), + ("textLength", "textLength"), + (None, None), + ], + default="string", + max_length=20, + ), + ), + migrations.AlterField( + model_name="occurrencereportbulkimporttask", + name="processing_status", + field=models.CharField( + choices=[ + ("queued", "Queued"), + ("started", "Started"), + ("failed", "Failed"), + ("completed", "Completed"), + ("archived", "Archived"), + ], + default="queued", + max_length=20, + ), + ), + ] diff --git a/boranga/migrations/0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more.py b/boranga/migrations/0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more.py new file mode 100644 index 00000000..26259714 --- /dev/null +++ b/boranga/migrations/0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 5.0.8 on 2024-08-19 03:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0432_occurrencereportbulkimportschema_datetime_created_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="occurrencereportbulkimportschemacolumn", + name="schema", + ), + migrations.RemoveField( + model_name="occurrencereportbulkimportschemacolumn", + name="import_content_type", + ), + migrations.RemoveField( + model_name="occurrencereportbulkimportschemacolumn", + name="list_lookup_class", + ), + migrations.DeleteModel( + name="OccurrenceReportBulkImportSchema", + ), + migrations.DeleteModel( + name="OccurrenceReportBulkImportSchemaColumn", + ), + ] diff --git a/boranga/migrations/0434_alter_communitydocument__file_and_more.py b/boranga/migrations/0434_alter_communitydocument__file_and_more.py new file mode 100644 index 00000000..a28fcbee --- /dev/null +++ b/boranga/migrations/0434_alter_communitydocument__file_and_more.py @@ -0,0 +1,56 @@ +# Generated by Django 5.0.8 on 2024-08-19 03:42 + +import datetime +import django.core.files.storage +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "boranga", + "0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more", + ), + ] + + operations = [ + migrations.CreateModel( + name="OccurrenceReportBulkImportSchema", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("version", models.IntegerField(default=1)), + ("datetime_created", models.DateTimeField(auto_now_add=True)), + ( + "datetime_updated", + models.DateTimeField(default=datetime.datetime.now), + ), + ( + "group_type", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="boranga.grouptype", + ), + ), + ], + options={ + "verbose_name": "Occurrence Report Bulk Import Schema", + "verbose_name_plural": "Occurrence Report Bulk Import Schemas", + }, + ), + migrations.AddConstraint( + model_name="occurrencereportbulkimportschema", + constraint=models.UniqueConstraint( + fields=("group_type", "version"), name="unique_schema_version" + ), + ), + ] diff --git a/boranga/migrations/0435_alter_communitydocument__file_and_more.py b/boranga/migrations/0435_alter_communitydocument__file_and_more.py new file mode 100644 index 00000000..266a1646 --- /dev/null +++ b/boranga/migrations/0435_alter_communitydocument__file_and_more.py @@ -0,0 +1,101 @@ +# Generated by Django 5.0.8 on 2024-08-19 03:59 + +import django.core.files.storage +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0434_alter_communitydocument__file_and_more"), + ("contenttypes", "0002_remove_content_type_name"), + ] + + operations = [ + migrations.CreateModel( + name="OccurrenceReportBulkImportSchemaColumn", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("django_import_field_name", models.CharField(max_length=50)), + ("xlsx_column_header_name", models.CharField(max_length=50)), + ( + "xlsx_data_validation_type", + models.CharField( + choices=[ + ("custom", "custom"), + ("whole", "whole"), + (None, None), + ("list", "list"), + ("time", "time"), + ("textLength", "textLength"), + ("decimal", "decimal"), + ("date", "date"), + ], + default="string", + max_length=20, + ), + ), + ( + "xlsx_data_validation_allow_blank", + models.BooleanField(default=False), + ), + ( + "xlsx_data_validation_operator", + models.CharField( + choices=[ + ("lessThan", "lessThan"), + ("greaterThan", "greaterThan"), + ("greaterThanOrEqual", "greaterThanOrEqual"), + ("equal", "equal"), + ("notEqual", "notEqual"), + ("lessThanOrEqual", "lessThanOrEqual"), + ("notBetween", "notBetween"), + ("between", "between"), + (None, None), + ], + default="between", + max_length=20, + ), + ), + ( + "xlsx_data_validation_formula1", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "xlsx_data_validation_formula2", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "django_import_content_type", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + related_name="import_columns", + to="contenttypes.contenttype", + ), + ), + ( + "schema", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="columns", + to="boranga.occurrencereportbulkimportschema", + ), + ), + ], + options={ + "verbose_name": "Occurrence Report Bulk Import Schema Column", + "verbose_name_plural": "Occurrence Report Bulk Import Schema Columns", + }, + ), + ] diff --git a/boranga/migrations/0436_alter_occurrencereportbulkimportschema_options_and_more.py b/boranga/migrations/0436_alter_occurrencereportbulkimportschema_options_and_more.py new file mode 100644 index 00000000..e88a899c --- /dev/null +++ b/boranga/migrations/0436_alter_occurrencereportbulkimportschema_options_and_more.py @@ -0,0 +1,60 @@ +# Generated by Django 5.0.8 on 2024-08-19 07:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0435_alter_communitydocument__file_and_more"), + ] + + operations = [ + migrations.AlterModelOptions( + name="occurrencereportbulkimportschema", + options={ + "ordering": ["group_type", "-version"], + "verbose_name": "Occurrence Report Bulk Import Schema", + "verbose_name_plural": "Occurrence Report Bulk Import Schemas", + }, + ), + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="xlsx_data_validation_operator", + field=models.CharField( + blank=True, + choices=[ + ("notEqual", "notEqual"), + ("between", "between"), + ("greaterThanOrEqual", "greaterThanOrEqual"), + ("lessThan", "lessThan"), + ("lessThanOrEqual", "lessThanOrEqual"), + ("equal", "equal"), + ("greaterThan", "greaterThan"), + ("notBetween", "notBetween"), + (None, None), + ], + max_length=20, + null=True, + ), + ), + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="xlsx_data_validation_type", + field=models.CharField( + blank=True, + choices=[ + ("list", "list"), + ("whole", "whole"), + ("custom", "custom"), + ("textLength", "textLength"), + ("time", "time"), + ("date", "date"), + ("decimal", "decimal"), + (None, None), + ], + max_length=20, + null=True, + ), + ), + ] diff --git a/boranga/migrations/0437_occurrencereportbulkimporttask_schema_and_more.py b/boranga/migrations/0437_occurrencereportbulkimporttask_schema_and_more.py new file mode 100644 index 00000000..2ed60554 --- /dev/null +++ b/boranga/migrations/0437_occurrencereportbulkimporttask_schema_and_more.py @@ -0,0 +1,63 @@ +# Generated by Django 5.0.8 on 2024-08-19 07:40 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0436_alter_occurrencereportbulkimportschema_options_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="occurrencereportbulkimporttask", + name="schema", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="boranga.occurrencereportbulkimportschema", + ), + ), + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="xlsx_data_validation_operator", + field=models.CharField( + blank=True, + choices=[ + ("notBetween", "notBetween"), + ("lessThan", "lessThan"), + ("greaterThan", "greaterThan"), + ("between", "between"), + ("notEqual", "notEqual"), + ("lessThanOrEqual", "lessThanOrEqual"), + ("equal", "equal"), + ("greaterThanOrEqual", "greaterThanOrEqual"), + (None, None), + ], + max_length=20, + null=True, + ), + ), + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="xlsx_data_validation_type", + field=models.CharField( + blank=True, + choices=[ + ("textLength", "textLength"), + ("custom", "custom"), + ("decimal", "decimal"), + ("whole", "whole"), + ("date", "date"), + ("list", "list"), + ("time", "time"), + (None, None), + ], + max_length=20, + null=True, + ), + ), + ] diff --git a/boranga/migrations/0438_alter_communitydocument__file_and_more.py b/boranga/migrations/0438_alter_communitydocument__file_and_more.py new file mode 100644 index 00000000..1e8f98fc --- /dev/null +++ b/boranga/migrations/0438_alter_communitydocument__file_and_more.py @@ -0,0 +1,52 @@ +# Generated by Django 5.0.8 on 2024-08-19 07:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boranga", "0437_occurrencereportbulkimporttask_schema_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="xlsx_data_validation_operator", + field=models.CharField( + blank=True, + choices=[ + ("between", "between"), + ("equal", "equal"), + ("greaterThan", "greaterThan"), + ("greaterThanOrEqual", "greaterThanOrEqual"), + ("lessThan", "lessThan"), + ("lessThanOrEqual", "lessThanOrEqual"), + ("notBetween", "notBetween"), + ("notEqual", "notEqual"), + (None, None), + ], + max_length=20, + null=True, + ), + ), + migrations.AlterField( + model_name="occurrencereportbulkimportschemacolumn", + name="xlsx_data_validation_type", + field=models.CharField( + blank=True, + choices=[ + ("custom", "custom"), + ("date", "date"), + ("decimal", "decimal"), + ("list", "list"), + ("textLength", "textLength"), + ("time", "time"), + ("whole", "whole"), + (None, None), + ], + max_length=20, + null=True, + ), + ), + ] From 6a9ffd4221c9dbb78297ecf2e80c65fde988bc06 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 20 Aug 2024 10:51:45 +0800 Subject: [PATCH 25/47] Add duplicate file contents checking. --- boranga/components/occurrence/models.py | 174 +++++++++++++----- boranga/components/occurrence/serializers.py | 10 +- .../internal/occurrence/bulk_import.vue | 2 +- 3 files changed, 138 insertions(+), 48 deletions(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 21da26ee..0e79e2fb 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -10,6 +10,7 @@ import openpyxl import pyproj import reversion +import xlrd from colorfield.fields import ColorField from django.conf import settings from django.contrib.contenttypes import fields @@ -5178,6 +5179,12 @@ def get_occurrence_report_bulk_import_path(instance, filename): class OccurrenceReportBulkImportTask(ArchivableModel): + schema = models.ForeignKey( + "OccurrenceReportBulkImportSchema", + on_delete=models.PROTECT, + null=True, + blank=True, + ) _file = models.FileField( upload_to=get_occurrence_report_bulk_import_path, max_length=512, @@ -5204,12 +5211,14 @@ class OccurrenceReportBulkImportTask(ArchivableModel): PROCESSING_STATUS_STARTED = "started" PROCESSING_STATUS_FAILED = "failed" PROCESSING_STATUS_COMPLETED = "completed" + PROCESSING_STATUS_ARCHIVED = "archived" PROCESSING_STATUS_CHOICES = ( (PROCESSING_STATUS_QUEUED, "Queued"), (PROCESSING_STATUS_STARTED, "Started"), (PROCESSING_STATUS_FAILED, "Failed"), (PROCESSING_STATUS_COMPLETED, "Completed"), + (PROCESSING_STATUS_ARCHIVED, "Archived"), ) processing_status = models.CharField( @@ -5225,7 +5234,13 @@ class Meta: def save(self, *args, **kwargs): if self._file: - self.file_hash = hashlib.sha256(self._file.read()).hexdigest() + logger.debug(f"Calculating hash for bulk import file {self._file.name}") + file_contents = self._file.read() + logger.debug(type(file_contents)) + self.file_hash = hashlib.sha256(file_contents).hexdigest() + logger.debug( + f"Hash for bulk import file {self._file.name}: {self.file_hash}" + ) super().save(*args, **kwargs) @property @@ -5260,10 +5275,11 @@ def total_time_taken_minues(self): @property def total_time_taken_human_readable(self): + if self.total_time_taken is None: + return None + if self.total_time_taken < 1: return "Less than a second" - if not self.total_time_taken: - return None if self.total_time_taken < 60: return f"{self.total_time_taken} seconds" @@ -5356,6 +5372,36 @@ def count_rows(self): logger.debug(f"Found {self.rows} rows in bulk import file {self._file.name}") self.save() + @classmethod + def validate_headers(self, _file, schema): + logger.info(f"Validating headers for bulk import task {self.id}") + + workbook = xlrd.open_workbook(file_contents=_file.read()) + + sheet = workbook.active + + headers = [cell.value for cell in sheet[1]] + + if not headers: + raise ValidationError("No headers found in the file") + + # Check that the headers match the schema (group type and version headings) + + schema_headers = self.schema.columns.all().values_list( + "xlsx_column_header_name", flat=True + ) + if headers == schema_headers: + return True + + extra_headers = set(headers) - set(schema_headers) + missing_headers = set(schema_headers) - set(headers) + error_string = f"Headers do not match schema: {self.schema}." + if missing_headers: + error_string += f" Missing: {missing_headers}" + if extra_headers: + error_string += f" Extra: {extra_headers}" + raise ValidationError(error_string) + def process(self): if self.processing_status == self.PROCESSING_STATUS_COMPLETED: logger.info(f"Bulk import task {self.id} has already been processed") @@ -5395,9 +5441,14 @@ def process(self): # Get the headers headers = [cell.value for cell in sheet[1]] + # TODO: Check that the headers match the schema (group type and version headings) + # Get the rows rows = list(sheet.iter_rows(min_row=2, max_row=self.rows + 1, values_only=True)) + # Occurrence reports to create + occurrence_reports = [] + # Process the rows for i, row in enumerate(rows): self.rows_processed = i + 1 @@ -5411,7 +5462,7 @@ def process(self): self.save() try: - self.process_row(row, headers) + self.process_row(row, headers, occurrence_reports) except Exception as e: logger.error(f"Error processing row {i + 1}: {e}") self.processing_status = ( @@ -5432,8 +5483,12 @@ def process(self): return - def process_row(self, row, headers): + def process_row(self, row, headers, occurrence_reports): row_hash = hashlib.sha256(str(row).encode()).hexdigest() + OccurrenceReport( + bulk_import_task=self, + import_hash=row_hash, + ) logger.info(f"Row hash: {row_hash}") return @@ -5446,21 +5501,56 @@ def retry(self): self.error_message = None self.save() + def revert(self): + # TODO: Using delete here due to the sheer number of records that could be created + # Still need to consider if we want to archive them + OccurrenceReport.objects.filter(bulk_import_task=self).delete() + + self.processing_status = self.PROCESSING_STATUS_ARCHIVED + self.archived = True + self.save() + class OccurrenceReportBulkImportSchema(models.Model): - name = models.CharField(max_length=255, blank=False, null=False) - version = models.IntegerField(default=1) group_type = models.ForeignKey( - GroupType, on_delete=models.PROTECT, null=True, blank=True + GroupType, on_delete=models.PROTECT, null=False, blank=False ) + version = models.IntegerField(default=1) + datetime_created = models.DateTimeField(auto_now_add=True) + datetime_updated = models.DateTimeField(default=datetime.now) class Meta: app_label = "boranga" verbose_name = "Occurrence Report Bulk Import Schema" verbose_name_plural = "Occurrence Report Bulk Import Schemas" + ordering = ["group_type", "-version"] + constraints = [ + models.UniqueConstraint( + fields=[ + "group_type", + "version", + ], + name="unique_schema_version", + ) + ] def __str__(self): - return f"{self.name} (Version: {self.version})" + return f"Group type: {self.group_type.name} (Version: {self.version})" + + @property + def preview_import_file(self): + workbook = openpyxl.Workbook() + worksheet = workbook.active + columns = self.columns.all() + if not columns.exists() or columns.count() == 0: + logger.warning( + f"No columns found for bulk import schema {self}. Returning empty preview file" + ) + return workbook + + headers = [column.xlsx_column_header_name for column in columns] + worksheet.append(headers) + return workbook class OccurrenceReportBulkImportSchemaColumn(models.Model): @@ -5469,36 +5559,49 @@ class OccurrenceReportBulkImportSchemaColumn(models.Model): related_name="columns", on_delete=models.CASCADE, ) - import_content_type = models.ForeignKey( + + # These two fields define where the data from the column will be imported to + django_import_content_type = models.ForeignKey( ct_models.ContentType, on_delete=models.PROTECT, null=True, blank=True, related_name="import_columns", ) - import_field_name = models.CharField(max_length=50, blank=False, null=False) + django_import_field_name = models.CharField(max_length=50, blank=False, null=False) - column_header_name = models.CharField(max_length=50, blank=False, null=False) - data_validation_type = models.CharField( + # The name of the column header in the .xlsx file + xlsx_column_header_name = models.CharField(max_length=50, blank=False, null=False) + + # The following fields are used to embed data validation in the .xlsx file + # so that the users can do a quick check before uploading + xlsx_data_validation_type = models.CharField( max_length=20, - choices=[(x, x) for x in DataValidation.type.values], - default="string", + choices=sorted( + [(x, x) for x in DataValidation.type.values], + key=lambda x: (x[0] is None, x), + ), + null=True, + blank=True, ) - required = models.BooleanField(default=False) - default_value = models.CharField(max_length=255, blank=True, null=True) - - max_length = models.IntegerField(null=True, blank=True) - min_value = models.IntegerField(null=True, blank=True) - max_value = models.IntegerField(null=True, blank=True) - - list_lookup_class = models.ForeignKey( - ct_models.ContentType, - on_delete=models.PROTECT, + xlsx_data_validation_allow_blank = models.BooleanField(default=False) + xlsx_data_validation_operator = models.CharField( + max_length=20, + choices=sorted( + [(x, x) for x in DataValidation.operator.values], + key=lambda x: (x[0] is None, x), + ), null=True, blank=True, - related_name="list_lookup_columns", ) - list_lookup_field = models.CharField(max_length=50, blank=True, null=True) + xlsx_data_validation_formula1 = models.CharField( + max_length=50, blank=True, null=True + ) + xlsx_data_validation_formula2 = models.CharField( + max_length=50, blank=True, null=True + ) + + # TODO: How are we going to do the list lookup validation for much larger datasets (mostly for species) class Meta: app_label = "boranga" @@ -5506,7 +5609,7 @@ class Meta: verbose_name_plural = "Occurrence Report Bulk Import Schema Columns" def __str__(self): - return f"{self.name} ({self.schema.name})" + return f"{self.xlsx_column_header_name} - {self.schema}" def validate(self, value): if self.data_validation_type == "whole": @@ -5515,15 +5618,6 @@ def validate(self, value): f"Default value for {self.column_header_name} must be an integer" ) - if self.min_value and value < self.min_value: - raise ValidationError( - f"Default value for {self.column_header_name} is too low" - ) - - if self.max_value and value > self.max_value: - raise ValidationError( - f"Default value for {self.column_header_name} is too high" - ) if self.data_validation_type == "decimal": try: value = Decimal(value) @@ -5546,12 +5640,6 @@ def validate(self, value): f"Default value for {self.column_header_name} must be a time" ) - if self.max_length: - if len(value) > self.max_length: - raise ValidationError( - f"Default value for {self.column_header_name} is too long" - ) - # Occurrence Report Document reversion.register(OccurrenceReportDocument) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index 74083de3..04c965d8 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -3891,25 +3891,27 @@ class Meta: ) def create(self, validated_data): - file_hash = hashlib.sha256(validated_data["_file"].read()).hexdigest() + _file = validated_data["_file"] + file_hash = hashlib.sha256(_file.read()).hexdigest() + _file.seek(0) qs = OccurrenceReportBulkImportTask.objects.filter(file_hash=file_hash) if qs.filter( processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED ).exists(): raise serializers.ValidationError( - "An import task with exactly same file contents has already been queued." + "An import task with exactly the same file contents has already been queued." ) if qs.filter( processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED ).exists(): raise serializers.ValidationError( - "An import task with exactly same file contents is already in progress." + "An import task with exactly the same file contents is already in progress." ) if qs.filter( processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_COMPLETED ).exists(): raise serializers.ValidationError( - "An import task with exactly same file contents has already been completed." + "An import task with exactly the same file contents has already been completed." ) return super().create(validated_data) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue index 7a76cb6a..4cfdd9c7 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue @@ -62,7 +62,7 @@ ref="bulk-import-file" aria-describedby="bulk-import-button" @change="bulkImportFileSelected" accept="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"> -
    +
    • {{ error }}
    From ce21edf965e16a9f1f61862a5e3971d13f448285 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 20 Aug 2024 15:18:25 +0800 Subject: [PATCH 26/47] Add new component to show list of bulk import schemas. --- .../occurrence/bulk_import_schema_list.vue | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue new file mode 100644 index 00000000..a13d9377 --- /dev/null +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue @@ -0,0 +1,100 @@ + + + From aaa436d92cb384db44c9b24bbf430150812852f0 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 20 Aug 2024 15:18:39 +0800 Subject: [PATCH 27/47] Some UI changes. --- .../occurrence/bulk_import_schema.vue | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue index 64ee6797..1b73e03c 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue @@ -15,21 +15,11 @@
    -

    Schema (Version: 1.0)

    -
    -
    -
    -
    - - -
    -
    +

    Flora Bulk Import Schema (Version: 1)

    @@ -42,7 +32,7 @@ - + - - - - - @@ -88,8 +70,21 @@
    -

    Scientific Name

    +
    + Selected Column Details +
    +
    + +
    +
    + +
    +
    +
    @@ -172,7 +167,7 @@ + role="button">
    From d4c725001cf2efe6c3d12014d55128dedf3e6cbd Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 20 Aug 2024 15:19:26 +0800 Subject: [PATCH 28/47] Make value returned by total_time_taken_human_readable better. Only create file hash if it hasn't already been created. --- boranga/components/occurrence/models.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 0e79e2fb..86b6acdc 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -5233,14 +5233,9 @@ class Meta: verbose_name_plural = "Occurrence Report Bulk Import Tasks" def save(self, *args, **kwargs): - if self._file: - logger.debug(f"Calculating hash for bulk import file {self._file.name}") - file_contents = self._file.read() - logger.debug(type(file_contents)) - self.file_hash = hashlib.sha256(file_contents).hexdigest() - logger.debug( - f"Hash for bulk import file {self._file.name}: {self.file_hash}" - ) + if not self.file_hash and self._file: + self._file.seek(0) + self.file_hash = hashlib.sha256(self._file.read()).hexdigest() super().save(*args, **kwargs) @property @@ -5284,7 +5279,11 @@ def total_time_taken_human_readable(self): if self.total_time_taken < 60: return f"{self.total_time_taken} seconds" if self.total_time_taken: - return f"{self.total_time_taken_minues} minutes" + whole_minutes = int(self.total_time_taken // 60) + remaining_seconds = round(self.total_time_taken - (whole_minutes * 60)) + if not remaining_seconds: + return f"{whole_minutes} minutes" + return f"{whole_minutes} minutes and {remaining_seconds} seconds" return None @property @@ -5375,7 +5374,6 @@ def count_rows(self): @classmethod def validate_headers(self, _file, schema): logger.info(f"Validating headers for bulk import task {self.id}") - workbook = xlrd.open_workbook(file_contents=_file.read()) sheet = workbook.active From 4fd3e10b23fe4dc5a4d1cd1fe755e64047109981 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Tue, 20 Aug 2024 15:20:12 +0800 Subject: [PATCH 29/47] Add route for bulk import schema list component and add bulk_import_schema_id param for the details page. --- .../boranga/src/components/internal/routes/index.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/routes/index.js b/boranga/frontend/boranga/src/components/internal/routes/index.js index ad6a7dae..f1c3d8c5 100755 --- a/boranga/frontend/boranga/src/components/internal/routes/index.js +++ b/boranga/frontend/boranga/src/components/internal/routes/index.js @@ -11,6 +11,7 @@ import Occurrence from '../occurrence/occurrence.vue' import OccurrenceReport from '../occurrence/occurrence_report.vue' import OccurrenceReportReferral from '../occurrence/referral.vue' import BulkImport from '../occurrence/bulk_import.vue' +import BulkImportSchemaList from '../occurrence/bulk_import_schema_list.vue' import BulkImportSchema from '../occurrence/bulk_import_schema.vue' export default @@ -66,10 +67,15 @@ export default }, children: [ { - path: 'bulk_import_schema/', - name: "occurrence-report-bulk-import-schema", + path: 'bulk_import_schema/:bulk_import_schema_id', + name: "occurrence-report-bulk-import-schema-details", component: BulkImportSchema }, + { + path: 'bulk_import_schema/', + name: "occurrence-report-bulk-import-schema-list", + component: BulkImportSchemaList + }, { path: 'bulk_import/', name: "occurrence-report-bulk-import", From 7f751b7b707e4072aa3f47be26b54571469e8ea2 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Wed, 21 Aug 2024 09:50:32 +0800 Subject: [PATCH 30/47] Replace health check, scheduler and azcopy installation with new centralised installation script. --- Dockerfile | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b0d10582..ecb7f918 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,16 +85,10 @@ RUN chmod 755 /startup.sh && \ mkdir /app && \ chown -R oim.oim /app && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ - wget https://raw.githubusercontent.com/dbca-wa/wagov_utils/main/wagov_utils/bin/health_check.sh -O /bin/health_check.sh && \ - chmod 755 /bin/health_check.sh && \ - wget https://raw.githubusercontent.com/dbca-wa/wagov_utils/main/wagov_utils/bin-python/scheduler/scheduler.py -O /bin/scheduler.py && \ - chmod 755 /bin/scheduler.py && \ - mkdir /tmp/azcopy/ && \ - wget https://aka.ms/downloadazcopy-v10-linux -O /tmp/azcopy/azcopy.tar.gz && \ - cd /tmp/azcopy/ ; tar -xzvf azcopy.tar.gz && \ - cp /tmp/azcopy/azcopy_linux_amd64_10.*/azcopy /bin/azcopy && \ - chmod 755 /bin/azcopy && \ - rm -rf /tmp/azcopy/ + wget https://raw.githubusercontent.com/dbca-wa/wagov_utils/main/wagov_utils/bin/default_script_installer.sh -O /tmp/default_script_installer.sh && \ + chmod 755 /tmp/default_script_installer.sh && \ + /tmp/default_script_installer.sh && \ + rm -rf /tmp/* FROM configure_boranga as python_dependencies_boranga From 90722a07045c06b142b70b11d3a500eb01a44989 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:07:30 +0800 Subject: [PATCH 31/47] get_openpyxl_data_validation_type_for_django_field function. --- boranga/helpers.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/boranga/helpers.py b/boranga/helpers.py index e885f7cd..8ee625dd 100755 --- a/boranga/helpers.py +++ b/boranga/helpers.py @@ -401,6 +401,32 @@ def get_instance_identifier(instance): ) +def get_openpyxl_data_validation_type_for_django_field(field): + from openpyxl.worksheet.datavalidation import DataValidation + + dv_types = dict(zip(DataValidation.type.values, DataValidation.type.values)) + + field_type_map = { + models.CharField: "textLength", + models.IntegerField: "whole", + models.DecimalField: "decimal", + models.BooleanField: "list", + models.DateField: "date", + models.DateTimeField: "date", + models.ForeignKey: "list", + } + + if isinstance(field, models.CharField) and field.choices: + return dv_types["list"] + + for django_field, dv_type in field_type_map.items(): + if isinstance(field, django_field): + return dv_types[dv_type] + + # Mainly covers TextField and other fields not explicitly handled + return None + + def clone_model( source_model_class: models.base.ModelBase, target_model_class: models.base.ModelBase, From 2326938f405227b60895da90abd1fb536d53dc57 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:08:09 +0800 Subject: [PATCH 32/47] Add ContentTypeViewSet. --- boranga/components/main/api.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/boranga/components/main/api.py b/boranga/components/main/api.py index 66b84b33..9914b171 100755 --- a/boranga/components/main/api.py +++ b/boranga/components/main/api.py @@ -3,16 +3,21 @@ import pyproj from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.core.cache import cache +from django_filters import rest_framework as filters +from rest_framework import filters as rest_framework_filters from rest_framework import viewsets from boranga import helpers from boranga.components.main.models import GlobalSettings, HelpTextEntry from boranga.components.main.serializers import ( + ContentTypeSerializer, GlobalSettingsSerializer, HelpTextEntrySerializer, ) from boranga.components.occurrence.models import Datum +from boranga.permissions import IsInternal logger = logging.getLogger(__name__) @@ -41,6 +46,15 @@ def get_queryset(self): return qs +class ContentTypeViewSet(viewsets.ReadOnlyModelViewSet): + queryset = ContentType.objects.filter(app_label="boranga") + serializer_class = ContentTypeSerializer + permission_classes = [IsInternal] + filter_backends = [filters.DjangoFilterBackend, rest_framework_filters.SearchFilter] + filterset_fields = ["app_label", "model"] + search_fields = ["^model"] + + class RetrieveActionLoggingViewsetMixin: """Mixin to automatically log user actions when a user retrieves an instance. From 0e0aec4fc4c47dc24defa4e0d0f3dc024dc7ff84 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:08:34 +0800 Subject: [PATCH 33/47] Add ContentTypeSerializer. --- boranga/components/main/serializers.py | 44 +++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/boranga/components/main/serializers.py b/boranga/components/main/serializers.py index 1dec4269..a75ef6cb 100755 --- a/boranga/components/main/serializers.py +++ b/boranga/components/main/serializers.py @@ -1,3 +1,4 @@ +from django.contrib.contenttypes.models import ContentType from ledger_api_client.ledger_models import EmailUserRO from ledger_api_client.ledger_models import EmailUserRO as EmailUser from rest_framework import serializers @@ -7,7 +8,10 @@ GlobalSettings, HelpTextEntry, ) -from boranga.helpers import is_django_admin +from boranga.helpers import ( + get_openpyxl_data_validation_type_for_django_field, + is_django_admin, +) class CommunicationLogEntrySerializer(serializers.ModelSerializer): @@ -109,3 +113,41 @@ class Meta: def get_user_can_administer(self, obj): return is_django_admin(self.context["request"]) + + +class ContentTypeSerializer(serializers.ModelSerializer): + model_fields = serializers.SerializerMethodField() + model_verbose_name = serializers.SerializerMethodField() + + class Meta: + model = ContentType + fields = "__all__" + + def get_model_verbose_name(self, obj): + if not obj.model_class(): + return None + return obj.model_class()._meta.verbose_name.title() + + def get_model_fields(self, obj): + if not obj.model_class(): + return [] + fields = obj.model_class()._meta.get_fields() + return [ + { + "name": ( + field.verbose_name.title() + if hasattr(field, "verbose_name") + else field.name + ), + "type": str(type(field)).split(".")[-1].replace("'>", ""), + "allow_null": field.null if hasattr(field, "null") else None, + "max_length": ( + field.max_length if hasattr(field, "max_length") else None + ), + "xlsx_validation_type": get_openpyxl_data_validation_type_for_django_field( + field + ), + } + for field in fields + if field.name != "id" + ] From 3a8089c2ccb1c7ab782a16b56337a41b1a4787e5 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:08:58 +0800 Subject: [PATCH 34/47] Add content_types end point. --- boranga/frontend/boranga/src/api.js | 1 + 1 file changed, 1 insertion(+) diff --git a/boranga/frontend/boranga/src/api.js b/boranga/frontend/boranga/src/api.js index 06ef6513..5aed3129 100644 --- a/boranga/frontend/boranga/src/api.js +++ b/boranga/frontend/boranga/src/api.js @@ -15,6 +15,7 @@ module.exports = { compliances: "/api/compliances.json", conservation_status_documents: "/api/conservation_status_documents.json", conservation_status: "/api/conservation_status", + content_types: "/api/content_types/", countries: '/api/countries', cs_external_referee_invites: "/api/cs_external_referee_invites", cs_referrals: "/api/cs_referrals.json", From 92c1ae4dfef43e568941e036412b1222acc49208 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:09:46 +0800 Subject: [PATCH 35/47] Update preview_import_file method on OccurrenceReportBulkImportSchema model. --- boranga/components/occurrence/models.py | 135 ++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index 86b6acdc..a050f742 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -29,6 +29,9 @@ from ledger_api_client.ledger_models import EmailUserRO as EmailUser from ledger_api_client.managed_models import SystemGroup from multiselectfield import MultiSelectField +from openpyxl.styles import NamedStyle +from openpyxl.styles.fonts import Font +from openpyxl.utils import get_column_letter from openpyxl.worksheet.datavalidation import DataValidation from boranga import exceptions @@ -5548,8 +5551,137 @@ def preview_import_file(self): headers = [column.xlsx_column_header_name for column in columns] worksheet.append(headers) + + dv_types = dict(zip(DataValidation.type.values, DataValidation.type.values)) + dv_operators = dict( + zip(DataValidation.operator.values, DataValidation.operator.values) + ) + + # Add the data validation for each column + for index, column in enumerate(columns): + column_letter = get_column_letter(index + 1) + cell_range = f"{column_letter}2:{column_letter}1048576" # 1048576 is the maximum number of rows in Excel + + model_class = column.django_import_content_type.model_class() + if not hasattr(model_class, column.django_import_field_name): + raise ValidationError( + f"Model {model_class} does not have field {column.django_import_field_name}" + ) + model_field = model_class._meta.get_field(column.django_import_field_name) + logger.debug(f"model_field_type: {type(model_field)}") + dv = None + if isinstance(model_field, models.fields.CharField) and model_field.choices: + dv = DataValidation( + type=dv_types["list"], + allow_blank=model_field.null, + formula1=",".join([c[0] for c in model_field.choices]), + error="Please select a valid option from the list", + errorTitle="Invalid selection", + prompt="Select a value from the list", + promptTitle="List selection", + ) + elif isinstance(model_field, models.fields.CharField): + dv = DataValidation( + type=dv_types["textLength"], + allow_blank=model_field.null, + operator=dv_operators["lessThanOrEqual"], + formula1=f"{model_field.max_length}", + error="Text must be less than or equal to {model_field.max_length} characters", + errorTitle="Text too long", + prompt=f"Maximum {model_field.max_length} characters", + promptTitle="Text length", + ) + elif isinstance( + model_field, (models.fields.DateTimeField, models.fields.DateField) + ): + dv = DataValidation( + type=dv_types["date"], + operator=dv_operators["greaterThanOrEqual"], + formula1="1900-01-01", + allow_blank=model_field.null, + error="Please enter a valid date", + errorTitle="Invalid date", + prompt="Enter a date", + promptTitle="Date", + ) + if isinstance(model_field, models.fields.DateTimeField): + date_style = NamedStyle( + name="datetime", number_format="DD/MM/YYYY HH:MM:MM" + ) + for cell in worksheet[column_letter]: + cell.style = date_style + elif isinstance(model_field, models.fields.IntegerField): + dv = DataValidation( + type=dv_types["whole"], + allow_blank=model_field.null, + error="Please enter a whole number", + errorTitle="Invalid number", + prompt="Enter a whole number", + promptTitle="Whole number", + ) + elif isinstance(model_field, models.fields.DecimalField): + dv = DataValidation( + type=dv_types["decimal"], + allow_blank=model_field.null, + error="Please enter a decimal number", + errorTitle="Invalid number", + prompt="Enter a decimal number", + promptTitle="Decimal number", + ) + elif isinstance(model_field, models.fields.BooleanField): + dv = DataValidation( + type=dv_types["list"], + allow_blank=model_field.null, + formula1='"True,False"', + error="Please select True or False", + errorTitle="Invalid selection", + prompt="Select True or False", + promptTitle="Boolean selection", + ) + else: + # Most covers TextField + # Postgresql Text field can handle up to 65,535 characters, .xlsx can handle 32,767 characters + # We'll gleefully assume this won't be an issue and not add a data validation for text fields =D + continue + + dv.showErrorMessage = True + worksheet.add_data_validation(dv) + dv.add(cell_range) + + # Make the headers bold + for cell in worksheet["A0:ZZ0"][0]: + cell.font = Font(bold=True) + + # Make the column widths appropriate + dims = {} + for row in worksheet.rows: + for cell in row: + if cell.value: + dims[cell.column] = ( + max((dims.get(cell.column, 0), len(str(cell.value)))) + 2 + ) + 2 + for col, value in dims.items(): + worksheet.column_dimensions[get_column_letter(col)].width = value + return workbook + def copy(self): + new_schema = OccurrenceReportBulkImportSchema( + group_type=self.group_type, + version=self.version + 1, + ) + new_schema.save() + + for column in self.columns.all(): + new_column = OccurrenceReportBulkImportSchemaColumn.objects.get( + pk=column.pk + ) + new_column.pk = None + new_column.schema = new_schema + new_column.save() + + return new_schema + class OccurrenceReportBulkImportSchemaColumn(models.Model): schema = models.ForeignKey( @@ -5718,3 +5850,6 @@ def validate(self, value): "identification", ], ) + +reversion.register(OccurrenceReportGeometry) +reversion.register(OccurrenceGeometry) From c3e7a536222eaa2436f4b287a6ab24e6a93f4a51 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:11:37 +0800 Subject: [PATCH 36/47] Register route for ContentTypeViewSet. --- boranga/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boranga/urls.py b/boranga/urls.py index 7ddcd9be..816d2e38 100755 --- a/boranga/urls.py +++ b/boranga/urls.py @@ -188,7 +188,7 @@ def trigger_error(request): router.register( r"help_text_entries", main_api.HelpTextEntryViewSet, "help_text_entries" ) - +router.register(r"content_types", main_api.ContentTypeViewSet, "content_types") router.registry.sort(key=lambda x: x[0]) api_patterns = [ From 579fea075c63c7a19fbba7c545f3bdbf27d99038 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:11:56 +0800 Subject: [PATCH 37/47] Updates to OccurrenceReportBulkImportSchemaSerializer. --- boranga/components/occurrence/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index 04c965d8..7a5e8ac7 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -3925,7 +3925,11 @@ class Meta: class OccurrenceReportBulkImportSchemaSerializer(serializers.ModelSerializer): - columns = OccurrenceReportBulkImportSchemaColumnSerializer(many=True) + columns = OccurrenceReportBulkImportSchemaColumnSerializer( + many=True, read_only=True + ) + group_type_display = serializers.CharField(source="group_type.name", read_only=True) + version = serializers.CharField(read_only=True) class Meta: model = OccurrenceReportBulkImportSchema From fc4090272b2a4578d852888084094625fe306a83 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:14:59 +0800 Subject: [PATCH 38/47] OccurrenceReportBulkImportSchemaViewSet changes: set version for new schema in perform_create method. Add custom actions for copy and save_column. --- boranga/components/occurrence/api.py | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/boranga/components/occurrence/api.py b/boranga/components/occurrence/api.py index 8497324c..6b907141 100644 --- a/boranga/components/occurrence/api.py +++ b/boranga/components/occurrence/api.py @@ -137,6 +137,7 @@ OccurrenceDocumentSerializer, OccurrenceLogEntrySerializer, OccurrenceReportAmendmentRequestSerializer, + OccurrenceReportBulkImportSchemaColumnSerializer, OccurrenceReportBulkImportSchemaSerializer, OccurrenceReportBulkImportTaskSerializer, OccurrenceReportDocumentSerializer, @@ -6276,6 +6277,8 @@ class OccurrenceReportBulkImportSchemaViewSet( queryset = OccurrenceReportBulkImportSchema.objects.all() serializer_class = OccurrenceReportBulkImportSchemaSerializer permission_classes = [OccurrenceReportBulkImportPermission] + filter_backends = [filters.DjangoFilterBackend] + filterset_fields = ["group_type"] def get_queryset(self): qs = self.queryset @@ -6283,6 +6286,18 @@ def get_queryset(self): qs = OccurrenceReportBulkImportSchema.objects.none() return qs + def perform_create(self, serializer): + latest_version = ( + OccurrenceReportBulkImportSchema.objects.filter( + group_type=serializer.validated_data["group_type"] + ) + .order_by("-version") + .first() + .version + ) + serializer.save(version=latest_version + 1) + return super().perform_create(serializer) + @list_route(methods=["get"], detail=False) def get_schema_list_by_group_type(self, request, *args, **kwargs): group_type = request.GET.get("group_type", None) @@ -6309,3 +6324,25 @@ def preview_import_file(self, request, *args, **kwargs): response["Content-Disposition"] = f"attachment; filename={filename}" buffer.close() return response + + @detail_route(methods=["post"], detail=True) + def copy(self, request, *args, **kwargs): + instance = self.get_object() + new_instance = instance.copy() + serializer = OccurrenceReportBulkImportSchemaSerializer(new_instance) + return Response(serializer.data, status=status.HTTP_201_CREATED) + + @detail_route(methods=["put"], detail=True) + def save_column(self, request, *args, **kwargs): + instance = self.get_object() + column_data = request.data.get("column_data", None) + if not column_data: + raise serializers.ValidationError("Column data is required") + serializer = OccurrenceReportBulkImportSchemaColumnSerializer( + instance, data=column_data + ) + serializer.is_valid(raise_exception=True) + serializer.save() + + serializer = OccurrenceReportBulkImportSchemaSerializer(instance) + return Response(serializer.data, status=status.HTTP_201_CREATED) From c3a84b8e2dfccacbe8a3e0edc987156a077e1eb6 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Thu, 22 Aug 2024 12:16:38 +0800 Subject: [PATCH 39/47] Add features for copying and creating new schemas. --- .../occurrence/bulk_import_schema_list.vue | 71 ++++++++++++++++--- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue index a13d9377..62ac33d7 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue @@ -12,17 +12,15 @@
    - +
    @@ -49,7 +47,7 @@ Edit - +
    @@ -70,7 +68,8 @@ export default { name: 'OccurrenceReportBulkImportSchemaList', data() { return { - group_type: null, + groupTypes: null, + groupType: null, bulkSchemas: null, } }, @@ -78,23 +77,73 @@ export default { alert }, methods: { + fetchGroupTypes() { + this.$http.get(api_endpoints.group_types_dict) + .then(response => { + this.groupTypes = response.data + }) + .catch(error => { + console.error(error) + }) + }, fetchBulkSchemas() { - if (!this.group_type) { + if (!this.groupType) { this.bulkSchemas = null return } - this.$http.get(api_endpoints.occurrence_report_bulk_import_schemas_by_group_type, { + this.$http.get(api_endpoints.occurrence_report_bulk_import_schemas, { params: { - group_type: this.group_type + group_type: this.groupType } }) .then(response => { - this.bulkSchemas = response.data + this.bulkSchemas = response.data.results + }) + .catch(error => { + console.error(error) + }) + }, + copySchema(id) { + this.$http.post(`${api_endpoints.occurrence_report_bulk_import_schemas}${id}/copy/`) + .then(response => { + if (response.status === 201) { + swal.fire({ + title: 'Success', + text: 'Schema copied successfully', + icon: 'success', + showConfirmButton: false, + timer: 1500 + }) + this.$router.push(`/internal/occurrence_report/bulk_import_schema/${response.data.id}`) + } + }) + .catch(error => { + console.error(error) + }) + }, + createNewVersion() { + this.$http.post(api_endpoints.occurrence_report_bulk_import_schemas, { + group_type: this.groupType + }) + .then(response => { + if (response.status === 201) { + swal.fire({ + title: 'Success', + text: 'New version created successfully', + icon: 'success', + showConfirmButton: false, + timer: 1500 + }) + this.$router.push(`/internal/occurrence_report/bulk_import_schema/${response.data.id}`) + } }) .catch(error => { console.error(error) }) } }, + created() { + this.fetchGroupTypes() + } } From b92c92c30900b234404d37ac5c476e1e8d7be4f4 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 14:42:45 +0800 Subject: [PATCH 40/47] Update pre-commit. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c67c97d..32ac72b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,13 +10,13 @@ repos: - id: check-yaml - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.17.0 hooks: - id: pyupgrade args: [--py310-plus] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black @@ -27,7 +27,7 @@ repos: args: ["--profile", "black"] - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 args: ["--config=setup.cfg","--per-file-ignores=boranga/settings.py:F405,E402","--ignore=E203, W503"] From 704b2366be4e295237f0761b3930e4c8c45bbe3d Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 15:59:34 +0800 Subject: [PATCH 41/47] Allow updates on OccurrenceReportBulkImportSchemaViewSet. --- boranga/components/occurrence/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/boranga/components/occurrence/api.py b/boranga/components/occurrence/api.py index 6b907141..cbb62a3d 100644 --- a/boranga/components/occurrence/api.py +++ b/boranga/components/occurrence/api.py @@ -6272,6 +6272,7 @@ class OccurrenceReportBulkImportSchemaViewSet( viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.CreateModelMixin, + mixins.UpdateModelMixin, mixins.ListModelMixin, ): queryset = OccurrenceReportBulkImportSchema.objects.all() From 4452130f26c145aeae25e498b8050a5bcbf21812 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 16:00:15 +0800 Subject: [PATCH 42/47] Add more data to filter_fields field on ContentTypeSerializer. --- boranga/components/main/serializers.py | 62 ++++++++++++++++++-------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/boranga/components/main/serializers.py b/boranga/components/main/serializers.py index a75ef6cb..2bc335d1 100755 --- a/boranga/components/main/serializers.py +++ b/boranga/components/main/serializers.py @@ -1,4 +1,7 @@ +import logging + from django.contrib.contenttypes.models import ContentType +from django.db.models.fields.related import ForeignKey, OneToOneField from ledger_api_client.ledger_models import EmailUserRO from ledger_api_client.ledger_models import EmailUserRO as EmailUser from rest_framework import serializers @@ -13,6 +16,8 @@ is_django_admin, ) +logger = logging.getLogger(__name__) + class CommunicationLogEntrySerializer(serializers.ModelSerializer): customer = serializers.PrimaryKeyRelatedField( @@ -132,22 +137,41 @@ def get_model_fields(self, obj): if not obj.model_class(): return [] fields = obj.model_class()._meta.get_fields() - return [ - { - "name": ( - field.verbose_name.title() - if hasattr(field, "verbose_name") - else field.name - ), - "type": str(type(field)).split(".")[-1].replace("'>", ""), - "allow_null": field.null if hasattr(field, "null") else None, - "max_length": ( - field.max_length if hasattr(field, "max_length") else None - ), - "xlsx_validation_type": get_openpyxl_data_validation_type_for_django_field( - field - ), - } - for field in fields - if field.name != "id" - ] + + def filter_fields(field): + return not field.auto_created and not ( + field.is_relation + and type(field) + not in [ + ForeignKey, + OneToOneField, + ] + ) + + fields = list(filter(filter_fields, fields)) + model_fields = [] + for field in fields: + display_name = ( + field.verbose_name.title() + if hasattr(field, "verbose_name") + else field.name + ) + field_type = str(type(field)).split(".")[-1].replace("'>", "") + choices = field.choices if hasattr(field, "choices") else None + allow_null = field.null if hasattr(field, "null") else None + max_length = field.max_length if hasattr(field, "max_length") else None + xlsx_validation_type = get_openpyxl_data_validation_type_for_django_field( + field + ) + model_fields.append( + { + "name": field.name, + "display_name": display_name, + "type": field_type, + "choices": choices, + "allow_null": allow_null, + "max_length": max_length, + "xlsx_validation_type": xlsx_validation_type, + } + ) + return model_fields From 9e49399978ba85bb6f8e87a5149f248b51b9be70 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 16:01:06 +0800 Subject: [PATCH 43/47] Add update method to OccurrenceReportBulkImportSchemaSerializer. --- boranga/components/occurrence/serializers.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/boranga/components/occurrence/serializers.py b/boranga/components/occurrence/serializers.py index 7a5e8ac7..93ab1a07 100644 --- a/boranga/components/occurrence/serializers.py +++ b/boranga/components/occurrence/serializers.py @@ -3925,9 +3925,7 @@ class Meta: class OccurrenceReportBulkImportSchemaSerializer(serializers.ModelSerializer): - columns = OccurrenceReportBulkImportSchemaColumnSerializer( - many=True, read_only=True - ) + columns = OccurrenceReportBulkImportSchemaColumnSerializer(many=True) group_type_display = serializers.CharField(source="group_type.name", read_only=True) version = serializers.CharField(read_only=True) @@ -3935,3 +3933,19 @@ class Meta: model = OccurrenceReportBulkImportSchema fields = "__all__" read_only_fields = ("id",) + + def update(self, instance, validated_data): + columns_data = validated_data.pop("columns") + # Delete any columns that are not in the new data + instance.columns.exclude( + id__in=[ + column_data["id"] + for column_data in columns_data + if hasattr(column_data, "id") + ] + ).delete() + for column_data in columns_data: + OccurrenceReportBulkImportSchemaColumn.objects.update_or_create( + **column_data + ) + return super().update(instance, validated_data) From f0b948c2d9697e4803b32a18dfef69e5f270f764 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 16:01:55 +0800 Subject: [PATCH 44/47] Comment fix. --- boranga/components/occurrence/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boranga/components/occurrence/models.py b/boranga/components/occurrence/models.py index a050f742..793a008d 100644 --- a/boranga/components/occurrence/models.py +++ b/boranga/components/occurrence/models.py @@ -5639,7 +5639,7 @@ def preview_import_file(self): promptTitle="Boolean selection", ) else: - # Most covers TextField + # Mostly covers TextField # Postgresql Text field can handle up to 65,535 characters, .xlsx can handle 32,767 characters # We'll gleefully assume this won't be an issue and not add a data validation for text fields =D continue From 0a0d6351a037eb61e30c454b93963c65ffb147b1 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 16:02:21 +0800 Subject: [PATCH 45/47] Minor change to sample help text. --- .../boranga/src/components/internal/occurrence/bulk_import.vue | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue index 4cfdd9c7..a848e6c4 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue @@ -8,8 +8,7 @@
    - Some information about the import process. Including a link to an example template - .xlsx file? + Some help text about the import process.
    From ab159ef3433cd58a04014cfebcd9a5f00a0177b6 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 16:03:39 +0800 Subject: [PATCH 46/47] Return None for validation type for foreign keys instead of list for now (May consider adding embedded list validation for foreign keys that are a basic lookup and only have say 30? or list items to choose from. --- boranga/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/boranga/helpers.py b/boranga/helpers.py index 8ee625dd..439e4da0 100755 --- a/boranga/helpers.py +++ b/boranga/helpers.py @@ -413,7 +413,6 @@ def get_openpyxl_data_validation_type_for_django_field(field): models.BooleanField: "list", models.DateField: "date", models.DateTimeField: "date", - models.ForeignKey: "list", } if isinstance(field, models.CharField) and field.choices: From ab65caa64692dfdbb750ecdb879227d76a92fc51 Mon Sep 17 00:00:00 2001 From: Oak McIlwain Date: Mon, 26 Aug 2024 16:04:00 +0800 Subject: [PATCH 47/47] Further work configuring the component. --- .../occurrence/bulk_import_schema.vue | 723 +++++++++++++----- 1 file changed, 541 insertions(+), 182 deletions(-) diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue index 1b73e03c..d123533d 100644 --- a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue +++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue @@ -1,6 +1,6 @@ @@ -215,31 +391,210 @@ export default { name: 'OccurrenceReportBulkImportSchema', data() { return { - + schema: null, + djangoContentTypes: null, + selectedColumn: null, + selectedContentType: null, + selectedField: null, + addEditMode: false, + newColumn: null, + saving: false, } }, components: { alert }, computed: { - + showValidationFields() { + return this.selectedColumn && + this.selectedColumn.xlsx_column_header_name && + this.selectedColumn.django_import_content_type && + this.selectedColumn.django_import_field_name && this.selectedField && this.selectedField.type === 'ForeignKey'; + }, }, methods: { + fetchBulkImportSchema() { + this.$http.get(`${api_endpoints.occurrence_report_bulk_import_schemas}${this.$route.params.bulk_import_schema_id}/`) + .then(response => { + this.schema = response.data + }) + .catch(error => { + console.error(error) + }) + }, + fetchContentTypes() { + this.$http.get(`${api_endpoints.content_types}`, { + params: { + app_label: 'boranga', + search: 'occurrencereport' + } + }) + .then(response => { + this.djangoContentTypes = response.data.results + this.$http.get(`${api_endpoints.content_types}`, { + params: { + app_label: 'boranga', + search: 'ocr' + } + }) + .then(response => { + this.djangoContentTypes.push(...response.data.results) + // this.removeAlreadySelectedFields(); + }) + .catch(error => { + console.error(error) + }) + }) + .catch(error => { + console.error(error) + }) + }, + removeAlreadySelectedFields() { + this.schema.columns.forEach(column => { + this.djangoContentTypes.forEach(djangoContentType => { + if (column.django_import_content_type == djangoContentType.id) { + djangoContentType.model_fields = djangoContentType.model_fields.filter( + modelField => modelField.name !== column.django_import_field_name + ) + } + }) + }) + }, + selectDjangoImportContentType() { + if (!this.selectedColumn.django_import_content_type) { + this.selectedField = null + this.selectedContentType = null + this.$refs['django-import-field'].focus() + return + } + this.selectedContentType = this.djangoContentTypes.filter( + djangoContentType => djangoContentType.id == this.selectedColumn.django_import_content_type + )[0] + this.$nextTick(() => { + this.enablePopovers(); + this.$refs['django-import-field'].focus() + }) + }, + selectDjangoImportField() { + if (!this.selectedColumn.django_import_field_name) { + this.selectedField = null + this.$refs['django-import-field'].focus() + return + } + this.selectedField = this.selectedContentType.model_fields.filter( + modelField => modelField.name == this.selectedColumn.django_import_field_name + )[0] + this.$nextTick(() => { + this.enablePopovers(); + if (!this.selectedColumn.id) { + this.selectedColumn.xlsx_column_header_name = this.selectedField.display_name + } + this.$refs['column-name'].focus() + }) + }, + getNewColumnData() { + return { + id: null, + schema: this.schema.id, + django_import_content_type: '', + django_import_field_name: '', + xlsx_column_header_name: '', + import_validations: [] + } + }, + addNewColumn() { + this.newColumn = Object.assign({}, this.getNewColumnData()) + this.schema.columns.push(this.newColumn) + this.selectedColumn = this.newColumn + this.addEditMode = true + // this.removeAlreadySelectedFields(); + this.$nextTick(() => { + this.enablePopovers(); + this.$refs['django-import-model'].focus() + }) + }, + selectColumn(column) { + this.addEditMode = true + this.selectedColumn = column + this.$nextTick(() => { + this.enablePopovers(); + if (this.selectedColumn.django_import_content_type) { + this.selectedContentType = this.djangoContentTypes.filter( + djangoContentType => djangoContentType.id == this.selectedColumn.django_import_content_type + )[0] + if (this.selectedColumn.django_import_field_name) { + this.selectedField = this.selectedContentType.model_fields.filter( + modelField => modelField.name == this.selectedColumn.django_import_field_name + )[0] + } + } + this.$refs['django-import-model'].focus() + }) + }, + cancelAddingColumn(column) { + this.schema.columns = this.schema.columns.filter(col => col !== column) + this.selectedColumn = null + this.addEditMode = false + }, + removeColumn(column) { + let columnTitle = column.xlsx_column_header_name ? `Are you sure you want to delete column: ${column.xlsx_column_header_name}?` : `Are you sure you want to delete this column?` + swal.fire({ + title: `Delete Column ${column.xlsx_column_header_name}`, + text: columnTitle, + icon: 'question', + showCancelButton: true, + confirmButtonText: 'Confirm Delete', + cancelButtonText: 'Cancel', + customClass: { + confirmButton: 'btn btn-primary', + cancelButton: 'btn btn-secondary me-2' + }, + reverseButtons: true + }).then((result) => { + if (result.isConfirmed) { + this.schema.columns = this.schema.columns.filter(column => column !== this.selectedColumn) + this.selectedColumn = null + this.addEditMode = false + if (column.id) { + this.save() + } + this.fetchContentTypes(); + } else { + if (this.selectedColumn) { + this.$refs['django-import-model'].focus() + } + } + }) + }, + save() { + this.saving = true; + this.$http.put(`${api_endpoints.occurrence_report_bulk_import_schemas}${this.schema.id}/`, this.schema) + .then(response => { + this.saving = false; + this.schema = response.data + }) + .catch(error => { + this.saving = false; + console.error(error) + }) + }, + saveAndExit() { + this.save() + this.$router.push(`/internal/occurrence_report/bulk_import_schema/`) + }, + enablePopovers() { + // enable all bootstrap 5 popovers + var popoverTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="popover"]')) + popoverTriggerList.map(function (popoverTriggerEl) { + return new bootstrap.Popover(popoverTriggerEl) + }) + } }, created() { - - }, - mounted() { - // enable all bootstrap 5 popovers - var popoverTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="popover"]')) - var popoverList = popoverTriggerList.map(function (popoverTriggerEl) { - return new bootstrap.Popover(popoverTriggerEl) - }) + this.fetchBulkImportSchema() + this.fetchContentTypes() }, - beforeDestroy() { - - } } @@ -252,4 +607,8 @@ div.scroll { overflow-y: hidden; white-space: nowrap; } + +tr.active { + background: rgba(51, 170, 51, .4) +}
    2 Scientific Name @@ -50,14 +40,6 @@
    2Scientific Name - -
    3 Observation Date