Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Display the export status during etl, #10238 #10248

Merged
merged 1 commit into from
Nov 10, 2023
Merged

Display the export status during etl, #10238 #10248

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions arches/app/etl_modules/base_excel_exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from datetime import datetime
from io import BytesIO
import json
import zipfile
from openpyxl.writer.excel import save_virtual_workbook
from django.core.files import File as DjangoFile
Expand Down Expand Up @@ -78,8 +79,8 @@ def export(self, request):

with connection.cursor() as cursor:
cursor.execute(
"""INSERT INTO load_event (loadid, complete, status, etl_module_id, load_start_time, user_id) VALUES (%s, %s, %s, %s, %s, %s)""",
(self.loadid, False, "validated", self.moduleid, datetime.now(), self.userid),
"""INSERT INTO load_event (loadid, complete, status, load_details, etl_module_id, load_start_time, user_id) VALUES (%s, %s, %s, %s, %s, %s, %s)""",
(self.loadid, False, "validated", json.dumps({"graph": graph_name}), self.moduleid, datetime.now(), self.userid),
)

if use_celery:
Expand Down
8 changes: 8 additions & 0 deletions arches/app/etl_modules/branch_excel_exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ def run_export_task(self, load_id, graph_id, graph_name, resource_ids, *args, **
resource_ids = [ row[0] for row in rows ]

with connection.cursor() as cursor:
cursor.execute(
"""UPDATE load_event SET load_details = %s WHERE loadid = (%s)""",
(json.dumps({
"graph": graph_name,
"number_of_resources": len(resource_ids),
}), load_id),
)

cursor.execute("""SELECT * FROM __get_nodegroup_tree_by_graph(%s)""", (graph_id,))
nodegroup_lookup = dictfetchall(cursor)

Expand Down
8 changes: 8 additions & 0 deletions arches/app/etl_modules/tile_excel_exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ def run_export_task(self, load_id, graph_id, graph_name, resource_ids, *args, **
resource_ids = [ row[0] for row in rows ]

with connection.cursor() as cursor:
cursor.execute(
"""UPDATE load_event SET load_details = %s WHERE loadid = (%s)""",
(json.dumps({
"graph": graph_name,
"number_of_resources": len(resource_ids),
}), load_id),
)

nodes = Node.objects.filter(graph_id=graph_id)
node_lookup_by_id = self.get_node_lookup_by_id(nodes)
tiles_to_export = {}
Expand Down
2 changes: 2 additions & 0 deletions arches/app/templates/javascript.htm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,8 @@
delete-resource-count='(resource) => {return {% trans "${resource} resources will be deleted" as deleteResourceCount %} `{{ deleteResourceCount|escapejs }}` }'
delete-tiles='{% trans "Delete Tiles" as deleteTiles %} "{{ deleteTiles|escapejs }}"'
delete-resources='{% trans "Delete Resources" as deleteResources %} "{{ deleteResources|escapejs }}"'
is-being-prepared='{% trans "is being prepared" as isBeingPrepared %} "{{ isBeingPrepared|escapejs }}"'
resources-are-being-exported='{% trans "resources are being exported" as resourcesAreBeingExported %} "{{ resourcesAreBeingExported|escapejs }}"'
tiles-to-be-deleted='{% trans "Tiles To Be Deleted" as tilesToBeDeleted %} "{{ tilesToBeDeleted|escapejs }}"'
no-data-to-delete='{% trans "No data to delete from the above selection" as noDataToDelete %} "{{ noDataToDelete|escapejs }}"'
following-resources-be-deleted='{% trans "The following resources will be deleted" as followingResourcesBeDeleted %} "{{ followingResourcesBeDeleted|escapejs }}"'
Expand Down
19 changes: 18 additions & 1 deletion arches/app/templates/views/components/etl_modules/branch-excel-exporter.htm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,22 @@ <h4 class="summary-title">
</h4>
<!-- ko if: ko.unwrap(loadDetails) -->
<div class="branch-csv-etl-load-summmary">
<!-- ko ifnot: ['failed','indexed'].includes(selectedLoadEvent().status) -->
<!-- ko if: loadDetails.graph && !loadDetails.number_of_resources -->
<div class="etl-loading-metadata-value">
<span data-bind="text: loadDetails.graph + ' ' + $root.translations.isBeingPrepared"></span>
<i class="fa fa-spin fa-spinner"></i>
</div>
<!-- /ko -->
<!-- ko if: loadDetails.number_of_resources -->
<div class="etl-loading-metadata-value">
<span data-bind="text: loadDetails.number_of_resources + ' ' + loadDetails.graph + ' ' + $root.translations.resourcesAreBeingExported"></span>
<i class="fa fa-spin fa-spinner"></i>
</div>
<!-- /ko -->
<!-- /ko -->

<!-- ko if: loadDetails.zipfile -->
<div class="summary-header">
<span data-bind="text: $root.translations.downloadExportedZipFile"></span>
</div>
Expand Down Expand Up @@ -104,8 +120,9 @@ <h4 class="summary-title">
<span class="etl-loading-metadata-key" data-bind="text: $root.translations.numberOfRelatedFilesInZip + ':'"></span>
<span class="etl-loading-metadata-value" data-bind="text: loadDetails.number_of_files"></span>
</div>
<!-- /ko -->
</div>
<!--/ko-->
<!-- /ko-->
</div>
{% endblock etl_status %}

Expand Down
19 changes: 18 additions & 1 deletion arches/app/templates/views/components/etl_modules/tile-excel-exporter.htm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,22 @@ <h4 class="summary-title">
</h4>
<!-- ko if: ko.unwrap(loadDetails) -->
<div class="branch-csv-etl-load-summmary">
<!-- ko ifnot: ['failed','indexed'].includes(selectedLoadEvent().status) -->
<!-- ko if: loadDetails.graph && !loadDetails.number_of_resources -->
<div class="etl-loading-metadata-value">
<span data-bind="text: loadDetails.graph + ' ' + $root.translations.isBeingPrepared"></span>
<i class="fa fa-spin fa-spinner"></i>
</div>
<!-- /ko -->
<!-- ko if: loadDetails.number_of_resources -->
<div class="etl-loading-metadata-value">
<span data-bind="text: loadDetails.number_of_resources + ' ' + loadDetails.graph + ' ' + $root.translations.resourcesAreBeingExported"></span>
<i class="fa fa-spin fa-spinner"></i>
</div>
<!-- /ko -->
<!-- /ko -->

<!-- ko if: loadDetails.zipfile -->
<div class="summary-header">
<span data-bind="text: $root.translations.downloadExportedZipFile"></span>
</div>
Expand Down Expand Up @@ -112,8 +128,9 @@ <h4 class="summary-title">
<span class="etl-loading-metadata-key" data-bind="text: $root.translations.numberOfRelatedFilesInZip + ':'"></span>
<span class="etl-loading-metadata-value" data-bind="text: loadDetails.number_of_files"></span>
</div>
<!-- /ko-->
</div>
<!--/ko-->
<!-- /ko-->
</div>
{% endblock etl_status %}

Expand Down
Loading