Skip to content

Commit

Permalink
Merge pull request #529 from oakdbca/ocr-bulk-import
Browse files Browse the repository at this point in the history 
Ocr bulk import
  • Loading branch information
@xzzy
xzzy authored Oct 17, 2024
2 parents 0ba713f + 95a6bf5 commit 377c0fa
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 12 deletions.
48 changes: 46 additions & 2 deletions boranga/components/occurrence/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,16 @@
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import IntegrityError, models, transaction
from django.db.models import CharField, Count, Func, ManyToManyField, Max, Q
from django.db.models import (
CharField,
Count,
Func,
ManyToManyField,
Max,
OuterRef,
Q,
Subquery,
)
from django.db.models.functions import Cast, Length
from django.utils import timezone
from django.utils.functional import cached_property
Expand Down Expand Up @@ -6872,11 +6881,34 @@ def get_sample_value(self, errors, species_or_community_identifier=None):
if isinstance(field, models.ForeignKey):
related_model_qs = self.filtered_related_model_qs

# Special case for species or community
# Ensure only species or communities that have occurrences are selected
# in case the schema includes the occurrence model (quite likely)
if field.name == "species":
related_model_qs = related_model_qs.annotate(
occurrence_count=Subquery(
Occurrence.objects.filter(species__pk=OuterRef("pk"))
.values("species")
.annotate(count=Count("id"))
.values("count")
)
).filter(occurrence_count__gt=0)
if field.name == "community":
related_model_qs = related_model_qs.annotate(
occurrence_count=Subquery(
Occurrence.objects.filter(community__pk=OuterRef("pk"))
.values("species")
.annotate(count=Count("id"))
.values("count")
)
).filter(occurrence_count__gt=0)

if not related_model_qs.exists():
error_message = f"No records found for foreign key field {field.related_model._meta.model_name}"
errors.append(
{
"error_type": "no_records",
"error_message": f"No records found for foreign key {field.related_model._meta.model_name}",
"error_message": error_message,
}
)

Expand Down Expand Up @@ -6987,6 +7019,18 @@ def get_sample_value(self, errors, species_or_community_identifier=None):
filter_field = {
"community__taxonomy__community_migrated_id": species_or_community_identifier
}
if not random_occurrence.exists():
error_message = (
f"No occurrences found where species or community identifier = "
f"{species_or_community_identifier}"
)
errors.append(
{
"error_type": "no_occurrences",
"error_message": error_message,
}
)
return None
return (
random_occurrence.filter(**filter_field)
.order_by("?")
Expand Down
36 changes: 26 additions & 10 deletions boranga/frontend/boranga/src/components/internal/occurrence/bulk_import_schema.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@
class="spinner-border spinner-border-sm me-1" role="status"
aria-hidden="true"></span><i v-else
class="bi bi-card-checklist me-1"></i>
Validat<template v-if="validatingSchema">ing</template><template v-else>e</template></button>
Validat<template v-if="validatingSchema">ing</template><template
v-else>e</template></button>
<a role="button" class="btn btn-primary"
:href="`/api/occurrence_report_bulk_import_schemas/${schema.id}/preview_import_file/?updated=${schema.datetime_updated}`"><i
class="bi bi-filetype-xlsx me-1"></i> Preview</a>
Expand Down Expand Up @@ -453,8 +454,8 @@
<template v-if="schema.can_user_edit && selectedColumn.is_editable_by_user">
<button
v-if="selectedColumn.django_import_content_type && selectedColumn.django_import_field_name"
class="btn btn-primary btn-sm me-2" @click.prevent="save()" :disabled="saving"><i
class="bi bi-floppy-fill me-1"></i>
class="btn btn-primary btn-sm me-2" @click.prevent="save()"
:disabled="saving"><i class="bi bi-floppy-fill me-1"></i>
Save
Column <template v-if="saving"><span
class="spinner-border spinner-border-sm ms-2" role="status"
Expand Down Expand Up @@ -491,6 +492,13 @@
</div>
</div>
</div>
<div v-else>
<div class="d-flex justify-content-center">
<div class="spinner-border text-primary" role="status">
<span class="visually-hidden">Loading...</span>
</div>
</div>
</div>
<div v-if="schema" class="row">
<div class="navbar fixed-bottom" style="background-color: rgb(245, 245, 245);">
<div class="container">
Expand All @@ -505,7 +513,9 @@
Continue <template v-if="saving"><span class="spinner-border spinner-border-sm ms-2"
role="status" aria-hidden="true"></span>
<span class="visually-hidden">Loading...</span></template></button><button
class="btn btn-primary me-2 float-end" @click.prevent="saveAndExit()" :disabled="saving">Save and Exit
class="btn btn-primary me-2 float-end" @click.prevent="saveAndExit()"
:disabled="saving">Save
and Exit
<template v-if="saving"><span class="spinner-border spinner-border-sm ms-2" role="status"
aria-hidden="true"></span>
<span class="visually-hidden">Loading...</span>
Expand Down Expand Up @@ -754,7 +764,8 @@ export default {
default_value: null,
import_validations: [],
lookup_filters: [],
is_editable_by_user: true
is_editable_by_user: true,
is_emailuser_column: false
}
},
addSingleColumn() {
Expand Down Expand Up @@ -971,16 +982,21 @@ export default {
} else if (Object.hasOwn(error, 'body')) {
errors = error.body
}
let error_message = 'Something went wrong :-('
if (errors instanceof Array) {
error_message = ''
let error_message_string = 'Something went wrong :-('
if (errors instanceof Object) {
error_message_string = ''
for (let i = 0; i < errors.length; i++) {
error_message += `<li class="mb-2">${errors[i].error_message}</li>`
let error_message = errors[i].error_message ? errors[i].error_message : errors[i]
error_message_string += `<li class="mb-2">${error_message}</li>`
}
console.log(error_message_string)
} else if (typeof errors === 'string') {
error_message_string = errors
}
console.error(error_message_string)
swal.fire({
title: 'Schema Validation Failed',
html: error_message,
html: error_message_string,
icon: 'error',
confirmButtonText: 'OK',
customClass: {
Expand Down

0 comments on commit 377c0fa

Please sign in to comment.