diff --git a/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue b/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue
index 52100e15..fb375c73 100644
--- a/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue
+++ b/boranga/frontend/boranga/src/components/common/occurrence_report_flora_dashboard.vue
@@ -60,6 +60,9 @@
+
Bulk Import
+
diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue
new file mode 100644
index 00000000..a848e6c4
--- /dev/null
+++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import.vue
@@ -0,0 +1,485 @@
+
+
+
+
+
+
{{ title }}
+
+
+
+
+ Some help text about the import process.
+
+
+
+
+
+
+
Queued Imports
+
+
+
+
+ Datetime Queued
+ |
+ File Name
+ |
+ File Size |
+ Row Count |
+ Time
+ Estimate |
+
+
+
+
+ {{ new Date(queuedImport.datetime_queued).toLocaleString() }} |
+ {{
+ queuedImport.file_name }} |
+ {{ queuedImport.file_size_megabytes }} MB |
+ {{ queuedImport.rows ? queuedImport.rows : 'Not
+ Counted' }} |
+ {{ queuedImport.estimated_processing_time_human_readable }} |
+
+
+
+
+
+
+
+
+
+
+
Currently Running Imports
+
+ Loading...
+
+
+
+
+
+ Datetime Started
+ |
+ File Name
+ |
+ File Size |
+ Progress
+ |
+
+
+
+
+ {{ new Date(currentlyRunningImport.datetime_started).toLocaleString() }}
+ |
+ {{ currentlyRunningImport.file_name }} |
+ {{ currentlyRunningImport.file_size_megabytes }} MB |
+
+
+ {{
+ currentlyRunningImport.percentage_complete }}%
+
+
+ |
+
+
+
+
+
+
+
+
+
+
+
Failed Bulk
+ Imports
+
+
+
+
+ Datetime Started |
+ File Name |
+ File Size |
+ Records Imported |
+ Actions |
+
+
+
+
+ {{ new Date(failedImport.datetime_started).toLocaleString() }} |
+ {{
+ failedImport.file_name }} |
+ {{ failedImport.file_size_megabytes }} MB |
+ {{ failedImport.rows ? failedImport.rows :
+ 'Not Counted' }} |
+
+
+
+ |
+
+
+
+
+
load more
+
+
+
+
+
+
+
+
+ {{ selectedErrors }}
+
+
+
+
+
+
+
+
+
+
Completed Bulk
+ Imports
+
+
+
+
+ Datetime
+ Completed |
+ File Name
+ |
+ Records Imported
+ |
+ Total
+ Time Taken |
+ Actions |
+
+
+
+
+ {{ new Date(completedImport.datetime_completed).toLocaleString() }}
+ |
+ {{
+ completedImport.file_name }} |
+ {{ completedImport.rows_processed
+ }} |
+ {{ completedImport.total_time_taken_human_readable }} |
+
+
+ |
+
+
+
+
+
load more
+
+
+
+
+
+
+
+
+
+
diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue
new file mode 100644
index 00000000..d123533d
--- /dev/null
+++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue
@@ -0,0 +1,614 @@
+
+
+
+
+
+
Occurrence Report Bulk Import Schema
+
+
+
+
+ Some help text about defining a bulk import schema
+
+
+
+
+
+
+
+
+
+
{{ schema.group_type_display }} Bulk Import
+ Schema Version: {{ schema.version
+ }}
+
+
+
+
Preview .xlsx File
+
+
+
+
+
+
+
+
+
+
+
+ Columns
+ (.xlsx)
+
+
+
+
+
+ {{ index +
+ 1 }}
+ |
+ {{
+ column.xlsx_column_header_name }}
+ |
+
+
+ |
+
+
+ Add Another Column |
+
+ |
+
+
+
+
+
+
+
+
+
+
+
+
+ Selected Column Details
+
+
+
+
+
+
+
+
+
+ No Column Selected
+
+
+
+ Select a column from the left panel
+ to view it's details
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue
new file mode 100644
index 00000000..62ac33d7
--- /dev/null
+++ b/boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema_list.vue
@@ -0,0 +1,149 @@
+
+
+
+
+
+
Occurrence Report Bulk Import Schemas
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Version |
+ Created |
+ Updated |
+ Actions |
+
+
+
+
+ {{ schema.version }} |
+ {{ new Date(schema.datetime_created).toLocaleDateString() }} {{ new
+ Date(schema.datetime_created).toLocaleTimeString() }} |
+ {{ new Date(schema.datetime_updated).toLocaleDateString() }} {{ new
+ Date(schema.datetime_updated).toLocaleTimeString() }} |
+
+ Edit
+
+ |
+
+
+
+
+
+
+
+
+
+
+
diff --git a/boranga/frontend/boranga/src/components/internal/routes/index.js b/boranga/frontend/boranga/src/components/internal/routes/index.js
index ab3dc67b..f1c3d8c5 100755
--- a/boranga/frontend/boranga/src/components/internal/routes/index.js
+++ b/boranga/frontend/boranga/src/components/internal/routes/index.js
@@ -10,6 +10,9 @@ import OccurrenceDash from '../occurrence/dashboard.vue'
import Occurrence from '../occurrence/occurrence.vue'
import OccurrenceReport from '../occurrence/occurrence_report.vue'
import OccurrenceReportReferral from '../occurrence/referral.vue'
+import BulkImport from '../occurrence/bulk_import.vue'
+import BulkImportSchemaList from '../occurrence/bulk_import_schema_list.vue'
+import BulkImportSchema from '../occurrence/bulk_import_schema.vue'
export default
{
@@ -63,6 +66,21 @@ export default
},
},
children: [
+ {
+ path: 'bulk_import_schema/:bulk_import_schema_id',
+ name: "occurrence-report-bulk-import-schema-details",
+ component: BulkImportSchema
+ },
+ {
+ path: 'bulk_import_schema/',
+ name: "occurrence-report-bulk-import-schema-list",
+ component: BulkImportSchemaList
+ },
+ {
+ path: 'bulk_import/',
+ name: "occurrence-report-bulk-import",
+ component: BulkImport
+ },
{
path: ':occurrence_report_id',
component: {
diff --git a/boranga/helpers.py b/boranga/helpers.py
index e885f7cd..439e4da0 100755
--- a/boranga/helpers.py
+++ b/boranga/helpers.py
@@ -401,6 +401,31 @@ def get_instance_identifier(instance):
)
+def get_openpyxl_data_validation_type_for_django_field(field):
+ from openpyxl.worksheet.datavalidation import DataValidation
+
+ dv_types = dict(zip(DataValidation.type.values, DataValidation.type.values))
+
+ field_type_map = {
+ models.CharField: "textLength",
+ models.IntegerField: "whole",
+ models.DecimalField: "decimal",
+ models.BooleanField: "list",
+ models.DateField: "date",
+ models.DateTimeField: "date",
+ }
+
+ if isinstance(field, models.CharField) and field.choices:
+ return dv_types["list"]
+
+ for django_field, dv_type in field_type_map.items():
+ if isinstance(field, django_field):
+ return dv_types[dv_type]
+
+ # Mainly covers TextField and other fields not explicitly handled
+ return None
+
+
def clone_model(
source_model_class: models.base.ModelBase,
target_model_class: models.base.ModelBase,
diff --git a/boranga/management/commands/ocr_pre_process_bulk_import_tasks.py b/boranga/management/commands/ocr_pre_process_bulk_import_tasks.py
new file mode 100644
index 00000000..c885dca7
--- /dev/null
+++ b/boranga/management/commands/ocr_pre_process_bulk_import_tasks.py
@@ -0,0 +1,43 @@
+import logging
+
+from django.core.management.base import BaseCommand
+
+from boranga.components.occurrence.models import OccurrenceReportBulkImportTask
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+ help = "Pre process the OCR bulk import tasks"
+
+ def handle(self, *args, **options):
+ logger.info(f"Running command {__name__}")
+
+ # Check if there are already any tasks running and return if so
+ if OccurrenceReportBulkImportTask.objects.filter(
+ processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED
+ ).exists():
+ logger.info("There is already a task running, returning")
+ return
+
+ # Get the next task to process
+ task = (
+ OccurrenceReportBulkImportTask.objects.filter(
+ processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED,
+ _file__isnull=False,
+ rows__isnull=True,
+ )
+ .order_by("datetime_queued")
+ .first()
+ )
+
+ if task is None:
+ logger.info("No tasks to process, returning")
+ return
+
+ # Process the task
+ task.count_rows()
+
+ logger.info(f"OCR Bulk Import Task {task.id} has {task.rows} rows.")
+
+ return
diff --git a/boranga/management/commands/ocr_process_bulk_import_queue.py b/boranga/management/commands/ocr_process_bulk_import_queue.py
new file mode 100644
index 00000000..ea434fa8
--- /dev/null
+++ b/boranga/management/commands/ocr_process_bulk_import_queue.py
@@ -0,0 +1,78 @@
+import logging
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from boranga.components.occurrence.models import OccurrenceReportBulkImportTask
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+ help = "Process the OCR bulk import queue"
+
+ def handle(self, *args, **options):
+ logger.info(f"Running command {__name__}")
+
+ # Check if there are any tasks that have been processing for too long
+ qs = OccurrenceReportBulkImportTask.objects.filter(
+ processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED,
+ datetime_started__lt=timezone.now()
+ - timezone.timedelta(seconds=settings.OCR_BULK_IMPORT_TASK_TIMEOUT_SECONDS),
+ )
+ if qs.exists():
+ for task in qs:
+ logger.info(
+ f"Task {task.id} has been processing for too long. Adding back to the queue"
+ )
+ task.processing_status = (
+ OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED
+ )
+ task.rows_processed = 0
+ task.save()
+
+ # Check if there are already any tasks running and return if so
+ if OccurrenceReportBulkImportTask.objects.filter(
+ processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_STARTED,
+ ).exists():
+ logger.info("There is already a task running, returning")
+ return
+
+ # Get the next task to process
+ task = (
+ OccurrenceReportBulkImportTask.objects.filter(
+ processing_status=OccurrenceReportBulkImportTask.PROCESSING_STATUS_QUEUED,
+ _file__isnull=False,
+ )
+ .order_by("datetime_queued")
+ .first()
+ )
+
+ if task is None:
+ logger.info("No tasks to process, returning")
+ return
+
+ try:
+ # Process the task
+ task.process()
+ except KeyboardInterrupt:
+ logger.info(f"OCR Bulk Import Task {task.id} was interrupted")
+ task.processing_status = (
+ OccurrenceReportBulkImportTask.PROCESSING_STATUS_FAILED
+ )
+ task.error_message = "KeyboardInterrupt"
+ task.save()
+ return
+ except Exception as e:
+ logger.error(f"Error processing OCR Bulk Import Task {task.id}: {e}")
+ task.processing_status = (
+ OccurrenceReportBulkImportTask.PROCESSING_STATUS_FAILED
+ )
+ task.error_message = str(e)
+ task.save()
+ return
+
+ logger.info(f"OCR Bulk Import Task {task.id} completed")
+
+ return
diff --git a/boranga/migrations/0428_occurrencereportbulkimporttask_and_more.py b/boranga/migrations/0428_occurrencereportbulkimporttask_and_more.py
new file mode 100644
index 00000000..4a82e33b
--- /dev/null
+++ b/boranga/migrations/0428_occurrencereportbulkimporttask_and_more.py
@@ -0,0 +1,70 @@
+# Generated by Django 5.0.8 on 2024-08-15 02:40
+
+import boranga.components.occurrence.models
+import django.core.files.storage
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0427_alter_community_renamed_from_and_more"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="OccurrenceReportBulkImportTask",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ (
+ "_file",
+ models.FileField(
+ max_length=512,
+ storage=django.core.files.storage.FileSystemStorage(
+ base_url="/private-media/",
+ location="/home/oak/dev/boranga/private-media/",
+ ),
+ upload_to=boranga.components.occurrence.models.get_occurrence_report_bulk_import_path,
+ ),
+ ),
+ ("rows", models.IntegerField(null=True)),
+ ("rows_processed", models.IntegerField(default=0)),
+ ("datetime_queued", models.DateTimeField(auto_now_add=True)),
+ ("datetime_started", models.DateTimeField(blank=True, null=True)),
+ ("datetime_completed", models.DateTimeField(blank=True, null=True)),
+ ("datetime_error", models.DateTimeField(blank=True, null=True)),
+ ("error_row", models.IntegerField(blank=True, null=True)),
+ ("error_message", models.TextField(blank=True, null=True)),
+ ("email_user", models.IntegerField()),
+ (
+ "processing_status",
+ models.CharField(
+ choices=[
+ ("queued", "Queued"),
+ ("started", "Started"),
+ ("failed", "Failed"),
+ ("completed", "Completed"),
+ ],
+ default="queued",
+ max_length=20,
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Occurrence Report Bulk Import Task",
+ "verbose_name_plural": "Occurrence Report Bulk Import Tasks",
+ },
+ ),
+ migrations.AlterModelOptions(
+ name="community",
+ options={"verbose_name_plural": "communities"},
+ ),
+ ]
diff --git a/boranga/migrations/0429_occurrencereport_bulk_import_task_and_more.py b/boranga/migrations/0429_occurrencereport_bulk_import_task_and_more.py
new file mode 100644
index 00000000..cd8329c4
--- /dev/null
+++ b/boranga/migrations/0429_occurrencereport_bulk_import_task_and_more.py
@@ -0,0 +1,30 @@
+# Generated by Django 5.0.8 on 2024-08-15 08:36
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0428_occurrencereportbulkimporttask_and_more"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="occurrencereport",
+ name="bulk_import_task",
+ field=models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="occurrence_reports",
+ to="boranga.occurrencereportbulkimporttask",
+ ),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimporttask",
+ name="rows",
+ field=models.IntegerField(editable=False, null=True),
+ ),
+ ]
diff --git a/boranga/migrations/0430_occurrencereport_import_hash_and_more.py b/boranga/migrations/0430_occurrencereport_import_hash_and_more.py
new file mode 100644
index 00000000..af3f612e
--- /dev/null
+++ b/boranga/migrations/0430_occurrencereport_import_hash_and_more.py
@@ -0,0 +1,66 @@
+# Generated by Django 5.0.8 on 2024-08-16 06:14
+
+import boranga.components.conservation_status.models
+import boranga.components.meetings.models
+import boranga.components.occurrence.models
+import boranga.components.species_and_communities.models
+import boranga.components.users.models
+import django.core.files.storage
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('boranga', '0429_occurrencereport_bulk_import_task_and_more'),
+ ('contenttypes', '0002_remove_content_type_name'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='occurrencereport',
+ name='import_hash',
+ field=models.CharField(blank=True, max_length=64, null=True),
+ ),
+ migrations.AddField(
+ model_name='occurrencereportbulkimporttask',
+ name='file_hash',
+ field=models.CharField(blank=True, max_length=64, null=True),
+ ),
+ migrations.CreateModel(
+ name='OccurrenceReportBulkImportSchema',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=255)),
+ ('version', models.IntegerField(default=1)),
+ ('group_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='boranga.grouptype')),
+ ],
+ options={
+ 'verbose_name': 'Occurrence Report Bulk Import Schema',
+ 'verbose_name_plural': 'Occurrence Report Bulk Import Schemas',
+ },
+ ),
+ migrations.CreateModel(
+ name='OccurrenceReportBulkImportSchemaColumn',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('import_field_name', models.CharField(max_length=50)),
+ ('column_header_name', models.CharField(max_length=50)),
+ ('data_validation_type', models.CharField(choices=[('textLength', 'textLength'), ('time', 'time'), ('list', 'list'), ('custom', 'custom'), (None, None), ('whole', 'whole'), ('decimal', 'decimal'), ('date', 'date')], default='string', max_length=20)),
+ ('required', models.BooleanField(default=False)),
+ ('default_value', models.CharField(blank=True, max_length=255, null=True)),
+ ('max_length', models.IntegerField(blank=True, null=True)),
+ ('min_value', models.IntegerField(blank=True, null=True)),
+ ('max_value', models.IntegerField(blank=True, null=True)),
+ ('list_lookup_field', models.CharField(blank=True, max_length=50, null=True)),
+ ('import_content_type', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='import_columns', to='contenttypes.contenttype')),
+ ('list_lookup_class', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='list_lookup_columns', to='contenttypes.contenttype')),
+ ('schema', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='columns', to='boranga.occurrencereportbulkimportschema')),
+ ],
+ options={
+ 'verbose_name': 'Occurrence Report Bulk Import Schema Column',
+ 'verbose_name_plural': 'Occurrence Report Bulk Import Schema Columns',
+ },
+ ),
+ ]
diff --git a/boranga/migrations/0431_occurrencereportbulkimporttask_archived_and_more.py b/boranga/migrations/0431_occurrencereportbulkimporttask_archived_and_more.py
new file mode 100644
index 00000000..c12bcbeb
--- /dev/null
+++ b/boranga/migrations/0431_occurrencereportbulkimporttask_archived_and_more.py
@@ -0,0 +1,36 @@
+# Generated by Django 5.0.8 on 2024-08-19 02:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0430_occurrencereport_import_hash_and_more"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="occurrencereportbulkimporttask",
+ name="archived",
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="data_validation_type",
+ field=models.CharField(
+ choices=[
+ ("textLength", "textLength"),
+ ("list", "list"),
+ ("time", "time"),
+ ("date", "date"),
+ ("whole", "whole"),
+ (None, None),
+ ("decimal", "decimal"),
+ ("custom", "custom"),
+ ],
+ default="string",
+ max_length=20,
+ ),
+ ),
+ ]
diff --git a/boranga/migrations/0432_occurrencereportbulkimportschema_datetime_created_and_more.py b/boranga/migrations/0432_occurrencereportbulkimportschema_datetime_created_and_more.py
new file mode 100644
index 00000000..f147efce
--- /dev/null
+++ b/boranga/migrations/0432_occurrencereportbulkimportschema_datetime_created_and_more.py
@@ -0,0 +1,61 @@
+# Generated by Django 5.0.8 on 2024-08-19 03:25
+
+import datetime
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0431_occurrencereportbulkimporttask_archived_and_more"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="occurrencereportbulkimportschema",
+ name="datetime_created",
+ field=models.DateTimeField(
+ auto_now_add=True, default=django.utils.timezone.now
+ ),
+ preserve_default=False,
+ ),
+ migrations.AddField(
+ model_name="occurrencereportbulkimportschema",
+ name="datetime_updated",
+ field=models.DateTimeField(default=datetime.datetime.now),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="data_validation_type",
+ field=models.CharField(
+ choices=[
+ ("whole", "whole"),
+ ("time", "time"),
+ ("decimal", "decimal"),
+ ("custom", "custom"),
+ ("list", "list"),
+ ("date", "date"),
+ ("textLength", "textLength"),
+ (None, None),
+ ],
+ default="string",
+ max_length=20,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimporttask",
+ name="processing_status",
+ field=models.CharField(
+ choices=[
+ ("queued", "Queued"),
+ ("started", "Started"),
+ ("failed", "Failed"),
+ ("completed", "Completed"),
+ ("archived", "Archived"),
+ ],
+ default="queued",
+ max_length=20,
+ ),
+ ),
+ ]
diff --git a/boranga/migrations/0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more.py b/boranga/migrations/0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more.py
new file mode 100644
index 00000000..26259714
--- /dev/null
+++ b/boranga/migrations/0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more.py
@@ -0,0 +1,31 @@
+# Generated by Django 5.0.8 on 2024-08-19 03:41
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0432_occurrencereportbulkimportschema_datetime_created_and_more"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="schema",
+ ),
+ migrations.RemoveField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="import_content_type",
+ ),
+ migrations.RemoveField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="list_lookup_class",
+ ),
+ migrations.DeleteModel(
+ name="OccurrenceReportBulkImportSchema",
+ ),
+ migrations.DeleteModel(
+ name="OccurrenceReportBulkImportSchemaColumn",
+ ),
+ ]
diff --git a/boranga/migrations/0434_alter_communitydocument__file_and_more.py b/boranga/migrations/0434_alter_communitydocument__file_and_more.py
new file mode 100644
index 00000000..a28fcbee
--- /dev/null
+++ b/boranga/migrations/0434_alter_communitydocument__file_and_more.py
@@ -0,0 +1,56 @@
+# Generated by Django 5.0.8 on 2024-08-19 03:42
+
+import datetime
+import django.core.files.storage
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ (
+ "boranga",
+ "0433_remove_occurrencereportbulkimportschemacolumn_schema_and_more",
+ ),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="OccurrenceReportBulkImportSchema",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("version", models.IntegerField(default=1)),
+ ("datetime_created", models.DateTimeField(auto_now_add=True)),
+ (
+ "datetime_updated",
+ models.DateTimeField(default=datetime.datetime.now),
+ ),
+ (
+ "group_type",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.PROTECT,
+ to="boranga.grouptype",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Occurrence Report Bulk Import Schema",
+ "verbose_name_plural": "Occurrence Report Bulk Import Schemas",
+ },
+ ),
+ migrations.AddConstraint(
+ model_name="occurrencereportbulkimportschema",
+ constraint=models.UniqueConstraint(
+ fields=("group_type", "version"), name="unique_schema_version"
+ ),
+ ),
+ ]
diff --git a/boranga/migrations/0435_alter_communitydocument__file_and_more.py b/boranga/migrations/0435_alter_communitydocument__file_and_more.py
new file mode 100644
index 00000000..266a1646
--- /dev/null
+++ b/boranga/migrations/0435_alter_communitydocument__file_and_more.py
@@ -0,0 +1,101 @@
+# Generated by Django 5.0.8 on 2024-08-19 03:59
+
+import django.core.files.storage
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0434_alter_communitydocument__file_and_more"),
+ ("contenttypes", "0002_remove_content_type_name"),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name="OccurrenceReportBulkImportSchemaColumn",
+ fields=[
+ (
+ "id",
+ models.AutoField(
+ auto_created=True,
+ primary_key=True,
+ serialize=False,
+ verbose_name="ID",
+ ),
+ ),
+ ("django_import_field_name", models.CharField(max_length=50)),
+ ("xlsx_column_header_name", models.CharField(max_length=50)),
+ (
+ "xlsx_data_validation_type",
+ models.CharField(
+ choices=[
+ ("custom", "custom"),
+ ("whole", "whole"),
+ (None, None),
+ ("list", "list"),
+ ("time", "time"),
+ ("textLength", "textLength"),
+ ("decimal", "decimal"),
+ ("date", "date"),
+ ],
+ default="string",
+ max_length=20,
+ ),
+ ),
+ (
+ "xlsx_data_validation_allow_blank",
+ models.BooleanField(default=False),
+ ),
+ (
+ "xlsx_data_validation_operator",
+ models.CharField(
+ choices=[
+ ("lessThan", "lessThan"),
+ ("greaterThan", "greaterThan"),
+ ("greaterThanOrEqual", "greaterThanOrEqual"),
+ ("equal", "equal"),
+ ("notEqual", "notEqual"),
+ ("lessThanOrEqual", "lessThanOrEqual"),
+ ("notBetween", "notBetween"),
+ ("between", "between"),
+ (None, None),
+ ],
+ default="between",
+ max_length=20,
+ ),
+ ),
+ (
+ "xlsx_data_validation_formula1",
+ models.CharField(blank=True, max_length=50, null=True),
+ ),
+ (
+ "xlsx_data_validation_formula2",
+ models.CharField(blank=True, max_length=50, null=True),
+ ),
+ (
+ "django_import_content_type",
+ models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ related_name="import_columns",
+ to="contenttypes.contenttype",
+ ),
+ ),
+ (
+ "schema",
+ models.ForeignKey(
+ on_delete=django.db.models.deletion.CASCADE,
+ related_name="columns",
+ to="boranga.occurrencereportbulkimportschema",
+ ),
+ ),
+ ],
+ options={
+ "verbose_name": "Occurrence Report Bulk Import Schema Column",
+ "verbose_name_plural": "Occurrence Report Bulk Import Schema Columns",
+ },
+ ),
+ ]
diff --git a/boranga/migrations/0436_alter_occurrencereportbulkimportschema_options_and_more.py b/boranga/migrations/0436_alter_occurrencereportbulkimportschema_options_and_more.py
new file mode 100644
index 00000000..e88a899c
--- /dev/null
+++ b/boranga/migrations/0436_alter_occurrencereportbulkimportschema_options_and_more.py
@@ -0,0 +1,60 @@
+# Generated by Django 5.0.8 on 2024-08-19 07:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0435_alter_communitydocument__file_and_more"),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name="occurrencereportbulkimportschema",
+ options={
+ "ordering": ["group_type", "-version"],
+ "verbose_name": "Occurrence Report Bulk Import Schema",
+ "verbose_name_plural": "Occurrence Report Bulk Import Schemas",
+ },
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="xlsx_data_validation_operator",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("notEqual", "notEqual"),
+ ("between", "between"),
+ ("greaterThanOrEqual", "greaterThanOrEqual"),
+ ("lessThan", "lessThan"),
+ ("lessThanOrEqual", "lessThanOrEqual"),
+ ("equal", "equal"),
+ ("greaterThan", "greaterThan"),
+ ("notBetween", "notBetween"),
+ (None, None),
+ ],
+ max_length=20,
+ null=True,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="xlsx_data_validation_type",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("list", "list"),
+ ("whole", "whole"),
+ ("custom", "custom"),
+ ("textLength", "textLength"),
+ ("time", "time"),
+ ("date", "date"),
+ ("decimal", "decimal"),
+ (None, None),
+ ],
+ max_length=20,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/boranga/migrations/0437_occurrencereportbulkimporttask_schema_and_more.py b/boranga/migrations/0437_occurrencereportbulkimporttask_schema_and_more.py
new file mode 100644
index 00000000..2ed60554
--- /dev/null
+++ b/boranga/migrations/0437_occurrencereportbulkimporttask_schema_and_more.py
@@ -0,0 +1,63 @@
+# Generated by Django 5.0.8 on 2024-08-19 07:40
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0436_alter_occurrencereportbulkimportschema_options_and_more"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="occurrencereportbulkimporttask",
+ name="schema",
+ field=models.ForeignKey(
+ blank=True,
+ null=True,
+ on_delete=django.db.models.deletion.PROTECT,
+ to="boranga.occurrencereportbulkimportschema",
+ ),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="xlsx_data_validation_operator",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("notBetween", "notBetween"),
+ ("lessThan", "lessThan"),
+ ("greaterThan", "greaterThan"),
+ ("between", "between"),
+ ("notEqual", "notEqual"),
+ ("lessThanOrEqual", "lessThanOrEqual"),
+ ("equal", "equal"),
+ ("greaterThanOrEqual", "greaterThanOrEqual"),
+ (None, None),
+ ],
+ max_length=20,
+ null=True,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="xlsx_data_validation_type",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("textLength", "textLength"),
+ ("custom", "custom"),
+ ("decimal", "decimal"),
+ ("whole", "whole"),
+ ("date", "date"),
+ ("list", "list"),
+ ("time", "time"),
+ (None, None),
+ ],
+ max_length=20,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/boranga/migrations/0438_alter_communitydocument__file_and_more.py b/boranga/migrations/0438_alter_communitydocument__file_and_more.py
new file mode 100644
index 00000000..1e8f98fc
--- /dev/null
+++ b/boranga/migrations/0438_alter_communitydocument__file_and_more.py
@@ -0,0 +1,52 @@
+# Generated by Django 5.0.8 on 2024-08-19 07:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("boranga", "0437_occurrencereportbulkimporttask_schema_and_more"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="xlsx_data_validation_operator",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("between", "between"),
+ ("equal", "equal"),
+ ("greaterThan", "greaterThan"),
+ ("greaterThanOrEqual", "greaterThanOrEqual"),
+ ("lessThan", "lessThan"),
+ ("lessThanOrEqual", "lessThanOrEqual"),
+ ("notBetween", "notBetween"),
+ ("notEqual", "notEqual"),
+ (None, None),
+ ],
+ max_length=20,
+ null=True,
+ ),
+ ),
+ migrations.AlterField(
+ model_name="occurrencereportbulkimportschemacolumn",
+ name="xlsx_data_validation_type",
+ field=models.CharField(
+ blank=True,
+ choices=[
+ ("custom", "custom"),
+ ("date", "date"),
+ ("decimal", "decimal"),
+ ("list", "list"),
+ ("textLength", "textLength"),
+ ("time", "time"),
+ ("whole", "whole"),
+ (None, None),
+ ],
+ max_length=20,
+ null=True,
+ ),
+ ),
+ ]
diff --git a/boranga/settings.py b/boranga/settings.py
index abf4e1a6..f98d440f 100755
--- a/boranga/settings.py
+++ b/boranga/settings.py
@@ -128,6 +128,7 @@ def show_toolbar(request):
"reversion_compare",
"nested_admin",
"colorfield",
+ "django_filters",
]
ADD_REVERSION_ADMIN = True
@@ -465,3 +466,7 @@ def show_toolbar(request):
# (_save method of FileSystemStorage class)
# As it causes a permission exception when using azure network drives
FILE_UPLOAD_PERMISSIONS = None
+
+OCR_BULK_IMPORT_TASK_TIMEOUT_SECONDS = env(
+ "OCR_BULK_IMPORT_TASK_TIMEOUT_SECONDS", 60 * 5
+) # Default = 5 minutes
diff --git a/boranga/static/boranga/css/base.css b/boranga/static/boranga/css/base.css
index f7f3141c..4ae2181d 100755
--- a/boranga/static/boranga/css/base.css
+++ b/boranga/static/boranga/css/base.css
@@ -129,20 +129,6 @@ body {
overflow-x: scroll;
}
-.modal-header {
- background-color: #529b6b;
- color: #fcb13f;
-}
-
-.modal-body {
- background-color: #529b6b;
- color: #fff;
-}
-
-.modal-footer {
- background-color: #529b6b;
-}
-
.popover {
max-width: 100%;
}
diff --git a/boranga/templates/webtemplate_dbca/includes/staff_menu.html b/boranga/templates/webtemplate_dbca/includes/staff_menu.html
index 07b1e69b..8d808e22 100644
--- a/boranga/templates/webtemplate_dbca/includes/staff_menu.html
+++ b/boranga/templates/webtemplate_dbca/includes/staff_menu.html
@@ -2,4 +2,7 @@
{% is_django_admin as is_django_admin_user %}
{% if is_django_admin_user %}
Admin
-{% endif %}
\ No newline at end of file
+{% endif %}
+{% if is_django_admin_user or request.user.is_superuser %}
+
OCR Bulk Import Schema
+{% endif %}
diff --git a/boranga/urls.py b/boranga/urls.py
index 0a4e3132..816d2e38 100755
--- a/boranga/urls.py
+++ b/boranga/urls.py
@@ -150,6 +150,16 @@ def trigger_error(request):
occurrence_api.OccurrenceReportPaginatedViewSet,
"occurrence_report_paginated",
)
+router.register(
+ r"occurrence_report_bulk_imports",
+ occurrence_api.OccurrenceReportBulkImportTaskViewSet,
+ "occurrence_report_bulk_imports",
+)
+router.register(
+ r"occurrence_report_bulk_import_schemas",
+ occurrence_api.OccurrenceReportBulkImportSchemaViewSet,
+ "occurrence_report_bulk_import_schemas",
+)
router.register(r"observer_detail", occurrence_api.ObserverDetailViewSet)
router.register(r"contact_detail", occurrence_api.ContactDetailViewSet)
router.register(r"occurrence_sites", occurrence_api.OccurrenceSiteViewSet)
@@ -178,7 +188,7 @@ def trigger_error(request):
router.register(
r"help_text_entries", main_api.HelpTextEntryViewSet, "help_text_entries"
)
-
+router.register(r"content_types", main_api.ContentTypeViewSet, "content_types")
router.registry.sort(key=lambda x: x[0])
api_patterns = [
diff --git a/boranga/views.py b/boranga/views.py
index 888c0f46..84f8f80a 100644
--- a/boranga/views.py
+++ b/boranga/views.py
@@ -27,15 +27,11 @@
from boranga.components.species_and_communities.models import Community, Species
from boranga.forms import LoginForm
from boranga.helpers import (
- is_conservation_status_assessor,
is_conservation_status_referee,
is_contributor,
is_django_admin,
is_internal,
- is_occurrence_approver,
- is_occurrence_assessor,
is_occurrence_report_referee,
- is_species_communities_approver,
)
logger = logging.getLogger(__name__)
@@ -209,10 +205,7 @@ def post(self, request):
def is_authorised_to_access_community_document(request, document_id):
if is_internal(request):
# check auth
- return (
- request.user.is_superuser
- or is_internal(request)
- )
+ return request.user.is_superuser or is_internal(request)
else:
return False
@@ -220,10 +213,7 @@ def is_authorised_to_access_community_document(request, document_id):
def is_authorised_to_access_species_document(request, document_id):
if is_internal(request):
# check auth
- return (
- request.user.is_superuser
- or is_internal(request)
- )
+ return request.user.is_superuser or is_internal(request)
else:
return False
@@ -231,10 +221,7 @@ def is_authorised_to_access_species_document(request, document_id):
def is_authorised_to_access_meeting_document(request, document_id):
if is_internal(request):
# check auth
- return (
- request.user.is_superuser
- or is_internal(request)
- )
+ return request.user.is_superuser or is_internal(request)
else:
return False
@@ -257,10 +244,7 @@ def is_authorised_to_access_occurrence_report_document(request, document_id):
if is_internal(request):
# check auth
- return (
- request.user.is_superuser
- or is_internal(request)
- )
+ return request.user.is_superuser or is_internal(request)
if is_occurrence_report_referee(request) and is_contributor(request):
file_name = get_file_name_from_path(request.path)
@@ -322,10 +306,7 @@ def is_authorised_to_access_occurrence_report_document(request, document_id):
def is_authorised_to_access_occurrence_document(request, document_id):
if is_internal(request):
# check auth
- return (
- request.user.is_superuser
- or is_internal(request)
- )
+ return request.user.is_superuser or is_internal(request)
else:
return False
@@ -336,10 +317,7 @@ def is_authorised_to_access_conservation_status_document(request, document_id):
if is_internal(request):
# check auth
- return (
- request.user.is_superuser
- or is_internal(request)
- )
+ return request.user.is_superuser or is_internal(request)
if is_conservation_status_referee(request) and is_contributor(request):
file_name = get_file_name_from_path(request.path)
@@ -378,7 +356,7 @@ def is_authorised_to_access_conservation_status_document(request, document_id):
document_id, request.path, referee_allowed_paths
)
- if is_contributor(request):
+ if is_contributor(request):
contributor_allowed_paths = ["documents", "amendment_request_documents"]
file_name = get_file_name_from_path(request.path)
return (
diff --git a/python-cron b/python-cron
index 3df1cba4..46b7d72a 100644
--- a/python-cron
+++ b/python-cron
@@ -8,3 +8,4 @@
# * * * * * Command
*/5 * * * * /app/venv/bin/python3 /app/manage.py runcrons >> /app/logs/cronjob.log 2>&1
+1 * * * * * /app/venv/bin/python3 /app/manage.py runcrons >> /app/logs/cronjob.log 2>&1
diff --git a/requirements.txt b/requirements.txt
index e4ed9174..337fbe1f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,6 +15,7 @@ git+https://github.com/xzzy/django-preserialize.git#egg=django-preserialize
django-countries~=7.5
django-cron==0.6.0 # This project is no longer maintained
django-dynamic-fixture==3.1.3
+django-filter~=24.3
gdal==3.8.4
openpyxl~=3.1
datapackage~=1.15