From 97c53623ef4161e4cc01709935fdd9ba1755d3ee Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 15 Jan 2025 17:04:54 +0530 Subject: [PATCH] fix(ingest/gc): fix logging --- .../ingestion/source/gc/soft_deleted_entity_cleanup.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py index 471eeff0224ed1..b3d6e8514c02f2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py @@ -99,7 +99,7 @@ class SoftDeletedEntitiesCleanupConfig(ConfigModel): @dataclass class SoftDeletedEntitiesReport(SourceReport): - num_queries_found: int = 0 + num_entities_found: Dict[str, int] = field(default_factory=dict) num_soft_deleted_entity_processed: int = 0 num_soft_deleted_retained_due_to_age: int = 0 num_soft_deleted_entity_removal_started: int = 0 @@ -277,7 +277,11 @@ def _get_soft_deleted(self, graphql_query: str, entity_type: str) -> Iterable[st # We make the batch size = config after call has succeeded once batch_size = self.config.batch_size scroll_id = scroll_across_entities.get("nextScrollId") - self.report.num_queries_found += scroll_across_entities.get("count") + if entity_type not in self.report.num_entities_found: + self.report.num_entities_found[entity_type] = 0 + self.report.num_entities_found[entity_type] += scroll_across_entities.get( + "count" + ) for query in scroll_across_entities.get("searchResults"): yield query["entity"]["urn"]