From 88b3893b5cca8ca04ce80cc347ed7b03c2a7ad15 Mon Sep 17 00:00:00 2001 From: Alisa Aylward Date: Wed, 21 Aug 2024 21:20:51 -0400 Subject: [PATCH] feat(ingest/snowflake): allow iceberg tables in lineage and access metadata (#10961) --- .../src/datahub/ingestion/source/snowflake/constants.py | 1 + .../src/datahub/ingestion/source/snowflake/snowflake_query.py | 1 + metadata-ingestion/tests/integration/snowflake/common.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py index 6f5e22e39d0c84..3ce684b29cf396 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/constants.py @@ -52,6 +52,7 @@ class SnowflakeObjectDomain(str, Enum): DATABASE = "database" SCHEMA = "schema" COLUMN = "column" + ICEBERG_TABLE = "iceberg table" GENERIC_PERMISSION_ERROR_KEY = "permission-error" diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index a2e18a64d9a809..662e1cc2509eae 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -35,6 +35,7 @@ class SnowflakeQuery: SnowflakeObjectDomain.EXTERNAL_TABLE.capitalize(), SnowflakeObjectDomain.VIEW.capitalize(), SnowflakeObjectDomain.MATERIALIZED_VIEW.capitalize(), + SnowflakeObjectDomain.ICEBERG_TABLE.capitalize(), } ACCESS_HISTORY_TABLE_VIEW_DOMAINS_FILTER = "({})".format( diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 5ef2eb420b8ed5..c4362368e28cd8 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -156,7 +156,7 @@ on basic_usage_counts.bucket_start_time = user_usage_counts.bucket_start_time and basic_usage_counts.object_name = user_usage_counts.object_name where - basic_usage_counts.object_domain in ('Table','External table','View','Materialized view') + basic_usage_counts.object_domain in ('Table','External table','View','Materialized view','Iceberg table') and basic_usage_counts.object_name is not null group by basic_usage_counts.object_name,