Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OCR Bulk Import Schema Changes #523

Merged
merged 8 commits into from
Aug 30, 2024
Merged

OCR Bulk Import Schema Changes #523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c639fbe
Make sure every schema has a migrated from id field.
oakdbca Aug 29, 2024
da903ab
Bug fix: Make sure version for schema is 1 if there are no other sche…
oakdbca Aug 29, 2024
39af6b2
Migration with constraints to enforce field and column name uniquenes…
oakdbca Aug 30, 2024
e35db07
Create a special serializer for the nested columns of the schema so w…
oakdbca Aug 30, 2024
ade3061
Add basic error reporting. Filter out fields that have already been a…
oakdbca Aug 30, 2024
eb7a7c8
Add comment.
oakdbca Aug 30, 2024
c1b158e
Tweak the functions that estimate processing time.
oakdbca Aug 30, 2024
1643156
Bug fix: Adjust perform create on OccurrenceReportBulkImportSchemaVie…
oakdbca Aug 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions boranga/components/occurrence/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6314,14 +6314,19 @@ def get_queryset(self):
return qs

def perform_create(self, serializer):
latest_version = (
OccurrenceReportBulkImportSchema.objects.filter(
group_type=serializer.validated_data["group_type"]
if OccurrenceReportBulkImportSchema.objects.filter(
group_type=serializer.validated_data["group_type"]
).exists():
latest_version = (
OccurrenceReportBulkImportSchema.objects.filter(
group_type=serializer.validated_data["group_type"]
)
.order_by("-version")
.first()
.version
)
.order_by("-version")
.first()
.version
)
else:
latest_version = 0
serializer.save(version=latest_version + 1)
return super().perform_create(serializer)

Expand Down
64 changes: 55 additions & 9 deletions boranga/components/occurrence/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5340,19 +5340,25 @@ def estimated_processing_time_seconds(self):
@property
def estimated_processing_time_minutes(self):
seconds = self.estimated_processing_time_seconds
if seconds:
return round(seconds / 60)
return None
if seconds is None:
return None

return round(seconds / 60)

@property
def estimated_processing_time_human_readable(self):
minutes = self.estimated_processing_time_minutes
seconds = self.estimated_processing_time_seconds

if not minutes:
if seconds is None:
return "No processing data available to estimate time"

if minutes == 0:
return "Less than a minute"
if seconds == 0:
return "Less than a second"

if seconds < 60:
return f"~{seconds} seconds"

minutes = self.estimated_processing_time_minutes

return f"~{minutes} minutes"

Expand Down Expand Up @@ -5541,6 +5547,22 @@ class Meta:
def __str__(self):
return f"Group type: {self.group_type.name} (Version: {self.version})"

def save(self, *args, **kwargs):
super().save(*args, **kwargs)
# Every schema should have a migrated_from_id column regardless if it is used
# for create new OCR records or updating existing ones
content_type = ct_models.ContentType.objects.get_for_model(OccurrenceReport)
if not self.columns.filter(
django_import_content_type=content_type,
django_import_field_name="migrated_from_id",
).exists():
OccurrenceReportBulkImportSchemaColumn.objects.create(
schema=self,
xlsx_column_header_name="OCR Migrated From ID",
django_import_content_type=content_type,
django_import_field_name="migrated_from_id",
)

@property
def preview_import_file(self):
workbook = openpyxl.Workbook()
Expand Down Expand Up @@ -5669,9 +5691,17 @@ def preview_import_file(self):
return workbook

def copy(self):
highest_version = OccurrenceReportBulkImportSchema.objects.filter(
if not self.pk:
raise ValueError("Schema must be saved before it can be copied")

if OccurrenceReportBulkImportSchema.objects.filter(
group_type=self.group_type
).aggregate(Max("version"))["version__max"]
).exists():
highest_version = OccurrenceReportBulkImportSchema.objects.filter(
group_type=self.group_type
).aggregate(Max("version"))["version__max"]
else:
highest_version = 0
new_schema = OccurrenceReportBulkImportSchema(
group_type=self.group_type,
version=highest_version + 1,
Expand Down Expand Up @@ -5749,6 +5779,22 @@ class Meta:
app_label = "boranga"
verbose_name = "Occurrence Report Bulk Import Schema Column"
verbose_name_plural = "Occurrence Report Bulk Import Schema Columns"
constraints = [
models.UniqueConstraint(
fields=[
"schema",
"django_import_content_type",
"django_import_field_name",
],
name="unique_schema_column_import",
violation_error_message="This field already exists in the schema",
),
models.UniqueConstraint(
fields=["schema", "xlsx_column_header_name"],
name="unique_schema_column_header",
violation_error_message="This column name already exists in the schema",
),
]

def __str__(self):
return f"{self.xlsx_column_header_name} - {self.schema}"
Expand Down
21 changes: 20 additions & 1 deletion boranga/components/occurrence/serializers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3925,10 +3925,24 @@ class Meta:
read_only_fields = ("id",)


class OccurrenceReportBulkImportSchemaColumnNestedSerializer(
serializers.ModelSerializer
):

class Meta:
model = OccurrenceReportBulkImportSchemaColumn
fields = "__all__"
read_only_fields = ("id",)
validators = [] # Validation is done in the parent serializer

def validate(self, attrs):
return super().validate(attrs)


class OccurrenceReportBulkImportSchemaSerializer(
TaggitSerializer, serializers.ModelSerializer
):
columns = OccurrenceReportBulkImportSchemaColumnSerializer(
columns = OccurrenceReportBulkImportSchemaColumnNestedSerializer(
many=True, allow_null=True, required=False
)
tags = TagListSerializerField(allow_null=True, required=False)
Expand All @@ -3940,6 +3954,10 @@ class Meta:
fields = "__all__"
read_only_fields = ("id",)

def validate(self, data):
logger.debug(f"data: {data}")
return data

def update(self, instance, validated_data):
columns_data = validated_data.pop("columns", None)

Expand All @@ -3955,6 +3973,7 @@ def update(self, instance, validated_data):
]
).delete()
for column_data in columns_data:
logger.debug(f"Column data: {column_data}")
OccurrenceReportBulkImportSchemaColumn.objects.update_or_create(
**column_data
)
Expand Down
60 changes: 43 additions & 17 deletions boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
<div class="mb-3">
<div class="card">
<div class="card-body">
<div class="mb-1">
<div class="container mb-0 pb-2">
<div class="border-bottom mb-2">
<div class="container mb-1 pb-2">
<div class="row">
<div class="col-5">
<h4 class="text-capitalize">{{ schema.group_type_display }} Bulk Import
Expand All @@ -25,7 +25,7 @@
<div class="col-7">
<div class="input-group float-start me-3" style="width:250px;">
<span class="input-group-text" id="basic-addon1"><i
class="bi bi-tag-fill text-secondary me-3"></i></span>
class="bi bi-tag-fill text-secondary"></i></span>
<input type="text" class="form-control form-control-sm"
placeholder="Add tag" @keydown="addTag" />
</div>
Expand All @@ -43,18 +43,19 @@
</div>
</div>
<div class="border-bottom mb-3">
<div class="container mb-1 pb-3">
<div class="row">
<div class="container mb-1 pb-2">
<div class="row align-items-center">
<label for="schema-name" class="col-sm-1 col-form-label">Name</label>
<div class="col-sm-4">
<input type="text" class="form-control" id="schema-name" ref="schema-name"
v-model="schema.name" aria-describedby="schema-name-help"
placeholder="Enter Schema Name" autofocus>
</div>
<label for="schema-tags" class="col-sm-1 col-form-label">Tags</label>
<div class="col-sm-6 d-flex flex-wrap align-items-center">
<div class="col-sm-6">
<template v-if="schema.tags && schema.tags.length > 0">
<span class="badge bg-info fs-6 me-2 mb-2"
<span
class="d-inline-flex align-items-center badge bg-info fs-6 my-2 me-2"
v-for="(tag, index) in schema.tags" :key="tag">{{ tag }}<i
class="bi bi-x-circle-fill ps-2" role="button"
@click="removeTag(index)"></i></span>
Expand All @@ -66,6 +67,11 @@
</div>
</div>
</div>
<div v-if="errors" class="mb-3">
<alert type="danger">
{{ errors}}
</alert>
</div>
<div class="mb-3">
<div class="container">
<div class="row">
Expand Down Expand Up @@ -153,13 +159,24 @@
v-model="selectedColumn.django_import_field_name"
@change="selectDjangoImportField">
<option value="">Select Django Model</option>
<option
v-for="modelField in selectedContentType.model_fields"
:value="modelField.name">
{{
modelField.display_name }} ({{
modelField.type }})
</option>
<template v-if="selectedColumn.id">
<option
v-for="modelField in selectedContentType.model_fields"
:value="modelField.name">
{{
modelField.display_name }} ({{
modelField.type }}) <template v-if="!modelField.allow_null">*</template>
</option>
</template>
<template v-else>
<option
v-for="modelField in djangoImportFieldsFiltered"
:value="modelField.name">
{{
modelField.display_name }} ({{
modelField.type }}) <template v-if="!modelField.allow_null">*</template>
</option>
</template>
</select>
</div>
</div>
Expand Down Expand Up @@ -252,7 +269,7 @@
class="">
<td>{{
choice[0]
}}
}}
</td>
</tr>
</tbody>
Expand Down Expand Up @@ -373,7 +390,8 @@
Column</button>
<button v-else class="btn btn-danger btn-sm"
@click.prevent="removeColumn(selectedColumn)"><i
class="bi bi-trash3-fill me-1"></i> Delete
class="bi bi-trash3-fill me-1"></i>
Delete
Column</button>
</div>
</form>
Expand Down Expand Up @@ -438,13 +456,15 @@ export default {
return {
schema: null,
djangoContentTypes: null,
djangoImportFieldsFiltered: null,
selectedColumn: null,
selectedColumnIndex: null,
selectedContentType: null,
selectedField: null,
addEditMode: false,
newColumn: null,
saving: false,
errors: null
}
},
components: {
Expand Down Expand Up @@ -516,6 +536,10 @@ export default {
this.selectedContentType = this.djangoContentTypes.filter(
djangoContentType => djangoContentType.id == this.selectedColumn.django_import_content_type
)[0]
// Filter out fields that are already columns in the schema
this.djangoImportFieldsFiltered = this.selectedContentType.model_fields.filter(
modelField => !this.schema.columns.some(column => column.django_import_field_name == modelField.name)
)
this.$nextTick(() => {
this.enablePopovers();
this.$refs['django-import-field'].focus()
Expand All @@ -532,8 +556,8 @@ export default {
)[0]
this.$nextTick(() => {
this.enablePopovers();
this.selectedColumn.xlsx_column_header_name = this.selectedField.display_name
if (!this.selectedColumn.id) {
this.selectedColumn.xlsx_column_header_name = this.selectedField.display_name
this.selectedColumn.xlsx_data_validation_allow_blank = this.selectedField.allow_null
}
this.$refs['column-name'].focus()
Expand Down Expand Up @@ -649,6 +673,7 @@ export default {
},
save() {
this.saving = true;
this.errors = null;
this.$http.put(`${api_endpoints.occurrence_report_bulk_import_schemas}${this.schema.id}/`, this.schema)
.then(response => {
this.saving = false;
Expand All @@ -659,6 +684,7 @@ export default {
})
.catch(error => {
this.saving = false;
this.errors = error.data
console.error(error)
})
},
Expand Down
34 changes: 34 additions & 0 deletions boranga/migrations/0441_alter_communitydocument__file_and_more.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Generated by Django 5.0.8 on 2024-08-29 23:59

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("boranga", "0440_occurrencereportbulkimportschema_name_and_more"),
("contenttypes", "0002_remove_content_type_name"),
]

operations = [
migrations.AddConstraint(
model_name="occurrencereportbulkimportschemacolumn",
constraint=models.UniqueConstraint(
fields=(
"schema",
"django_import_content_type",
"django_import_field_name",
),
name="unique_schema_column_import",
violation_error_message="This field already exists in the schema",
),
),
migrations.AddConstraint(
model_name="occurrencereportbulkimportschemacolumn",
constraint=models.UniqueConstraint(
fields=("schema", "xlsx_column_header_name"),
name="unique_schema_column_header",
violation_error_message="This column name already exists in the schema",
),
),
]
Loading