From b848274f500d24f5bd8efe53bddfefa136a88080 Mon Sep 17 00:00:00 2001 From: Joy A Date: Wed, 23 Oct 2024 12:59:12 +0530 Subject: [PATCH] avniproject/avni-media#179 | Media search performance improvements - Respect index configuration change on media table - Use alternate query when media is not filtered based on address to avoid expensive address join for orgs with lot of address levels --- .../etl/domain/metadata/ColumnMetadata.java | 4 ++ .../etl/repository/MediaTableRepository.java | 2 +- .../rowMappers/MediaTableMetadataBuilder.java | 4 +- .../sql/MediaSearchQueryBuilder.java | 6 ++ .../searchMediaWithoutAddressFilter.sql.st | 72 +++++++++++++++++++ 5 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 src/main/resources/sql/api/searchMediaWithoutAddressFilter.sql.st diff --git a/src/main/java/org/avniproject/etl/domain/metadata/ColumnMetadata.java b/src/main/java/org/avniproject/etl/domain/metadata/ColumnMetadata.java index 1444cd1..70f3bfc 100644 --- a/src/main/java/org/avniproject/etl/domain/metadata/ColumnMetadata.java +++ b/src/main/java/org/avniproject/etl/domain/metadata/ColumnMetadata.java @@ -79,6 +79,10 @@ public ColumnMetadata(Column column, Integer conceptId, ConceptType conceptType, this(null, column, conceptId, conceptType, conceptUuid, null); } + public ColumnMetadata(Column column) { + this(column, null, null, null); + } + public Integer getConceptId() { return conceptId; } diff --git a/src/main/java/org/avniproject/etl/repository/MediaTableRepository.java b/src/main/java/org/avniproject/etl/repository/MediaTableRepository.java index becfad7..7dee79f 100644 --- a/src/main/java/org/avniproject/etl/repository/MediaTableRepository.java +++ b/src/main/java/org/avniproject/etl/repository/MediaTableRepository.java @@ -75,7 +75,7 @@ private List searchInternal(MediaSearchRequest mediaSearchRequest, Page p conceptFilterSearches = determineConceptFilterTablesAndColumns(mediaSearchRequest.getConceptFilters()); } - Query query = new MediaSearchQueryBuilder() + Query query = new MediaSearchQueryBuilder(mediaSearchRequest) .withPage(page) .withMediaSearchRequest(mediaSearchRequest) .withSearchConceptFilters(conceptFilterSearches) diff --git a/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java b/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java index 7336aec..d516bcf 100644 --- a/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java +++ b/src/main/java/org/avniproject/etl/repository/rowMappers/MediaTableMetadataBuilder.java @@ -1,9 +1,11 @@ package org.avniproject.etl.repository.rowMappers; import org.avniproject.etl.domain.metadata.ColumnMetadata; +import org.avniproject.etl.domain.metadata.IndexMetadata; import org.avniproject.etl.domain.metadata.TableMetadata; import org.avniproject.etl.repository.rowMappers.tableMappers.MediaTable; +import java.util.Objects; import java.util.stream.Collectors; public class MediaTableMetadataBuilder { @@ -13,7 +15,7 @@ public static TableMetadata build() { mediaTableMetadata.setName(mediaTable.name(null)); mediaTableMetadata.setType(TableMetadata.Type.Media); mediaTableMetadata.addColumnMetadata(mediaTable.columns().stream().map(column -> new ColumnMetadata(column, null, null, null)).collect(Collectors.toList())); - + mediaTableMetadata.addIndexMetadata(mediaTable.columns().stream().map(column -> column.isIndexed() ? new IndexMetadata(new ColumnMetadata(column)) : null).filter(Objects::nonNull).collect(Collectors.toList())); return mediaTableMetadata; } } diff --git a/src/main/java/org/avniproject/etl/repository/sql/MediaSearchQueryBuilder.java b/src/main/java/org/avniproject/etl/repository/sql/MediaSearchQueryBuilder.java index 314fe35..38e2e57 100644 --- a/src/main/java/org/avniproject/etl/repository/sql/MediaSearchQueryBuilder.java +++ b/src/main/java/org/avniproject/etl/repository/sql/MediaSearchQueryBuilder.java @@ -15,6 +15,7 @@ public class MediaSearchQueryBuilder { private final ST template; private final Map parameters = new HashMap<>(); private final static String sqlTemplate = readFile("/sql/api/searchMedia.sql.st"); + private final static String withoutAddressFilterSqlTemplate = readFile("/sql/api/searchMediaWithoutAddressFilter.sql.st"); private static final Logger logger = Logger.getLogger(MediaSearchQueryBuilder.class); public MediaSearchQueryBuilder() { @@ -22,6 +23,11 @@ public MediaSearchQueryBuilder() { addDefaultParameters(); } + public MediaSearchQueryBuilder(MediaSearchRequest mediaSearchRequest) { + this.template = mediaSearchRequest.getAddresses().isEmpty() ? new ST(withoutAddressFilterSqlTemplate) : new ST(sqlTemplate); + addDefaultParameters(); + } + public MediaSearchQueryBuilder withMediaSearchRequest(MediaSearchRequest request) { template.add("request", request); addParameters(request); diff --git a/src/main/resources/sql/api/searchMediaWithoutAddressFilter.sql.st b/src/main/resources/sql/api/searchMediaWithoutAddressFilter.sql.st new file mode 100644 index 0000000..1ebf3da --- /dev/null +++ b/src/main/resources/sql/api/searchMediaWithoutAddressFilter.sql.st @@ -0,0 +1,72 @@ +SELECT *, row_to_json(address.*) as address FROM +( + SELECT media.id as id, + media.uuid as uuid, + media.subject_first_name as subject_first_name, + media.subject_last_name as subject_last_name, + media.subject_middle_name as subject_middle_name, + media.is_voided as is_voided, + media.created_by_id as created_by_id, + media.last_modified_by_id as last_modified_by_id, + media.created_date_time as created_date_time, + media.last_modified_date_time as last_modified_date_time, + media.organisation_id as organisation_id, + media.image_url as image_url, + media.sync_parameter_key1 as sync_parameter_key1, + media.sync_parameter_key2 as sync_parameter_key2, + media.sync_parameter_value1 as sync_parameter_value1, + media.sync_parameter_value2 as sync_parameter_value2, + media.subject_type_name as subject_type_name, + media.encounter_type_name as encounter_type_name, + media.program_name as program_name, + media.concept_name as concept_name, + media.address_id as address_id, + media.entity_id as entity_id + FROM .media media + + . ON media.entity_id = .id + + AND ."" + + IN ('}; separator="', '">') + + if AND else '' AND '' !> + + BETWEEN '' AND '' + + + AND ."" ILIKE '%%'}; separator=" \nAND "> + + }; separator="\n"> + + where media.image_url is not null + and media.is_voided is false + + and media.created_date_time >= :fromDate + and media.created_date_time \<= :toDate + and (%' or media.subject_middle_name ilike '%%' or media.subject_last_name ilike '%%') + }; separator="\n AND ">) + + and media.subject_type_name in (:subjectTypeNames) + and media.program_name in (:programNames) + and media.encounter_type_name in (:encounterTypeNames) + and media.concept_name in (:imageConcepts) + and ( + and media.sync_parameter_value1 in (:syncConceptValues_)) + or (media.sync_parameter_key2 = :syncConceptName_ and media.sync_parameter_value2 in (:syncConceptValues_)))}; + separator="\n OR " + > + ) + and ( + address." id" in (:addressLevelIds_)1 = 2}; separator="\n OR "> + ) + + ORDER BY media.created_date_time desc + LIMIT :limit OFFSET :offset +) media_search_result + JOIN .address address ON address.id = media_search_result.address_id + ORDER BY media_search_result.created_date_time desc +;