From fa4f5f277a5398edca8bd7a10d629ab07afaec25 Mon Sep 17 00:00:00 2001 From: Siddique Bagwan Date: Thu, 19 Dec 2024 09:24:58 +0530 Subject: [PATCH 01/13] lookml parameter support --- .../source/looker/looker_template_language.py | 31 ++- .../tests/integration/lookml/test_lookml.py | 1 + .../data.model.lkml | 4 + .../star_award_winner.view.lkml | 23 +++ .../vv_lineage_liquid_template_golden.json | 186 ++++++++++++++++++ 5 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index 6d49d57e07743..b1beb96b7eeda 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -198,9 +198,9 @@ def __init__(self, source_config: LookMLSourceConfig): def transform(self, view: dict) -> dict: value_to_transform: Optional[str] = None - # is_attribute_supported check is required because not all transformer works on all attributes in current - # case mostly all transformer works on sql_table_name and derived.sql attributes, - # however IncompleteSqlTransformer only transform the derived.sql attribute + # is_attribute_supported check is required because not all transformers work on all attributes in the current + # case, mostly all transformers work on sql_table_name and derived.sql attributes; + # however, IncompleteSqlTransformer only transform the derived.sql attribute if SQL_TABLE_NAME in view and self.is_attribute_supported(SQL_TABLE_NAME): # Give precedence to already processed transformed view.sql_table_name to apply more transformation value_to_transform = view.get( @@ -287,7 +287,7 @@ def _apply_transformation(self, value: str, view: dict) -> str: class DropDerivedViewPatternTransformer(LookMLViewTransformer): """ - drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values. + drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values. Example: transform ${employee_income_source.SQL_TABLE_NAME} to employee_income_source.SQL_TABLE_NAME """ @@ -335,6 +335,26 @@ def _apply_transformation(self, value: str, view: dict) -> str: return self._apply_regx(value) +class LookmlParameterTransformer(LookMLViewTransformer): + """ + Replace the lookml parameter @{variable} with their values. + """ + + PATTERN = r"@{(\w+)}" + + def resolve_lookml_parameter(self, text: str) -> str: + return re.sub( + LookmlParameterTransformer.PATTERN, + lambda match: self.source_config.liquid_variable.get( + match.group(1), match.group(0) + ), # Replace or keep original + text, + ) + + def _apply_transformation(self, value: str, view: dict) -> str: + return self.resolve_lookml_parameter(text=value) + + class TransformedLookMlView: """ TransformedLookMlView is collecting output of LookMLViewTransformer and creating a new transformed LookML view. @@ -401,6 +421,9 @@ def process_lookml_template_language( LiquidVariableTransformer( source_config=source_config ), # Now resolve liquid variables + LookmlParameterTransformer( + source_config=source_config + ), # Remove @{variable} with its corresponding value DropDerivedViewPatternTransformer( source_config=source_config ), # Remove any ${} symbol diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 940e7f36675f7..66b49cdb6eaaa 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -873,6 +873,7 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t "_is_selected": True, }, "source_region": "ap-south-1", + "star_award_winner_year": "public.winner_2025", } pipeline = Pipeline.create(new_recipe) diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml index d570e0ecdb5b2..ae6f70443ab56 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml @@ -10,6 +10,7 @@ include: "environment_activity_logs.view.lkml" include: "employee_income_source_as_per_env.view.lkml" include: "rent_as_employee_income_source.view.lkml" include: "child_view.view.lkml" +include: "star_award_winner.view.lkml" explore: activity_logs { } @@ -39,4 +40,7 @@ explore: rent_as_employee_income_source { } explore: child_view { +} + +explore: star_award_winner { } \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml new file mode 100644 index 0000000000000..e8f1a8aa48f95 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml @@ -0,0 +1,23 @@ +view: star_award_winner { + sql_table_name: @{star_award_winner_year} ;; + + dimension: id { + label: "id" + primary_key: yes + type: number + sql: ${TABLE}.id ;; + } + + parameter: star_award_winner_year { + type: string + allowed_value: { + label: "Star Award Winner Of 2025" + value: "public.winner_2025" + } + allowed_value: { + label: "Star Award Winner Of 2024" + value: "public.winner_2024" + } + } + +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json index 468450310c2ab..9af13f2a6b0a1 100644 --- a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -2827,6 +2827,192 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: star_award_winner {\n sql_table_name: @{star_award_winner_year} ;;\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n parameter: star_award_winner_year {\n type: string\n allowed_value: {\n label: \"Star Award Winner Of 2025\"\n value: \"public.winner_2025\"\n }\n allowed_value: {\n label: \"Star Award Winner Of 2024\"\n value: \"public.winner_2024\"\n }\n }\n\n}", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD),id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "star_award_winner", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "id", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": true + } + ], + "primaryKeys": [ + "id" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "star_award_winner.view.lkml", + "looker.model": "data" + }, + "name": "star_award_winner", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", From 865b5ffdd5eb6fa87e61633e18004dd3baf6ccae Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 8 Jan 2025 16:40:49 +0530 Subject: [PATCH 02/13] fix: pr comment --- .../source/looker/looker_template_language.py | 50 +++- .../ingestion/source/looker/looker_test.py | 215 ++++++++++++++++++ .../ingestion/source/looker/lookml_config.py | 9 + 3 files changed, 265 insertions(+), 9 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index b1beb96b7eeda..bd7fef8fee189 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -337,19 +337,51 @@ def _apply_transformation(self, value: str, view: dict) -> str: class LookmlParameterTransformer(LookMLViewTransformer): """ - Replace the lookml parameter @{variable} with their values. + Replace LookML parameters (@{param} or ${var}) from the configuration. """ - PATTERN = r"@{(\w+)}" + PARAMETER_PATTERN = r"@{(\w+)}" # Matches @{param} + LIQUID_VARIABLE_PATTERN = r"\${(\w+)}" # Matches ${var} def resolve_lookml_parameter(self, text: str) -> str: - return re.sub( - LookmlParameterTransformer.PATTERN, - lambda match: self.source_config.liquid_variable.get( - match.group(1), match.group(0) - ), # Replace or keep original - text, - ) + """ + Resolves LookML parameters (@{param}) and liquid variables (${var}). + Logs warnings for misplaced or missing variables. + """ + + def replace_parameters(match): + key = match.group(1) + # Check if it's a misplaced liquid variable + if key in self.source_config.liquid_variable: + logger.warning( + f"Misplaced liquid variable '@{{{key}}}' detected. Use '${{{key}}}' instead." + ) + return f"@{{{key}}}" + + # Resolve parameter + if key in self.source_config.lookml_parameter: + return str(self.source_config.lookml_parameter.get(key)) + + # Log warning for missing parameter + logger.warning(f"Parameter '@{{{key}}}' not found in configuration.") + return "" + + def replace_liquid_variables(match): + key = match.group(1) + # Resolve liquid variable + if key in self.source_config.liquid_variable: + return str(self.source_config.liquid_variable.get(key)) + + # Log warning for missing liquid variable + logger.warning(f"Liquid variable '${{{key}}}' not found in configuration.") + return "" + + # Resolve @{param} (parameters) + text = re.sub(self.PARAMETER_PATTERN, replace_parameters, text) + + # Resolve ${var} (liquid variables) + text = re.sub(self.LIQUID_VARIABLE_PATTERN, replace_liquid_variables, text) + return text def _apply_transformation(self, value: str, view: dict) -> str: return self.resolve_lookml_parameter(text=value) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py new file mode 100644 index 0000000000000..f9e30d5323d47 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py @@ -0,0 +1,215 @@ +import json +import logging +import os +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Sequence, Union + +import looker_sdk +import requests +from looker_sdk.sdk.api40.models import DashboardBase + +failed_dashboad_id_via_rest = {} +failed_dashboad_id_via_sdk = {} + + +def setup_logging(): + """Set up dual logging to console and file.""" + # Create a logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + # Create handlers for both file and console + file_handler = logging.FileHandler("looker_api_logs.log") + console_handler = logging.StreamHandler() + + # Set logging level for both handlers + file_handler.setLevel(logging.DEBUG) + console_handler.setLevel(logging.DEBUG) + + # Create a logging format + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") + file_handler.setFormatter(formatter) + console_handler.setFormatter(formatter) + + # Add handlers to the logger + logger.addHandler(file_handler) + logger.addHandler(console_handler) + + +def get_access_token(): + """Authenticate with Looker API and return the access token.""" + url = f"{os.getenv('LOOKERSDK_BASE_URL')}/api/4.0/login" + data = { + "client_id": os.getenv("LOOKERSDK_CLIENT_ID"), + "client_secret": os.getenv("LOOKERSDK_CLIENT_SECRET"), + } + response = requests.post(url, data=data) + if response.status_code == 200: + return response.json()["access_token"] + else: + logging.error("Failed to authenticate with Looker API") + logging.error(response.text) + raise Exception("Authentication failed") + + +def __fields_mapper(fields: Union[str, List[str]]) -> str: + """Helper method to turn single string or list of fields into Looker API compatible fields param""" + return fields if isinstance(fields, str) else ",".join(fields) + + +def fetch_dashboard_sdk(dashboard_id): + sdk = looker_sdk.init40() + logging.info( + f"---------------------- Fetching Dashboard ID {dashboard_id} Via SDK ----------------------" + ) + try: + fields = [ + "id", + "title", + "dashboard_elements", + "dashboard_filters", + "deleted", + "hidden", + "description", + "folder", + "user_id", + "created_at", + "updated_at", + "last_updater_id", + "deleted_at", + "deleter_id", + ] + dashboard = sdk.dashboard(dashboard_id, fields=__fields_mapper(fields)) + logging.info( + f"Dashboard ID {dashboard_id} details fetched successfully via SDK:" + ) + logging.info(json.dumps(dashboard, indent=2, default=str)) + except Exception as e: + logging.error(f"SDK Error for Dashboard ID {dashboard_id}: {e}") + failed_dashboad_id_via_sdk[dashboard_id] = str(e) + logging.info( + f"---------------------- Request Completed {dashboard_id} Via SDK----------------------" + ) + + +def fetch_dashboard(access_token, dashboard_id): + """Fetch a dashboard by ID using the Looker API and log with separators.""" + headers = { + "Authorization": f"token {access_token}", + "Content-Type": "application/json", + } + url = f"{os.getenv('LOOKERSDK_BASE_URL')}/api/4.0/dashboards/{dashboard_id}" + response = requests.get(url, headers=headers) + + logging.info( + f"---------------------- Fetching Dashboard ID {dashboard_id} ----------------------" + ) + + if response.status_code == 200: + logging.info(f"Dashboard ID {dashboard_id} details fetched successfully:") + logging.info(json.dumps(response.json(), indent=2)) + else: + logging.error(f"Failed to fetch Dashboard ID {dashboard_id}:") + value = response.content.decode(encoding="utf-8") + logging.error(value) + failed_dashboad_id_via_rest[dashboard_id] = response.text + logging.info( + f"---------------------- Request Completed {dashboard_id} ----------------------" + ) + + +def check_environment_variables(): + """Check if all required environment variables are set and log which are missing.""" + required_vars = [ + "LOOKERSDK_CLIENT_ID", + "LOOKERSDK_CLIENT_SECRET", + "LOOKERSDK_BASE_URL", + ] + missing_vars = [var for var in required_vars if not os.getenv(var)] + if missing_vars: + error_message = ( + f"Missing required environment variables: {', '.join(missing_vars)}" + ) + logging.error(error_message) + raise EnvironmentError(error_message) + + +def all_dashboards(fields: Union[str, List[str]]) -> Sequence[DashboardBase]: + sdk = looker_sdk.init40() + return sdk.all_dashboards(fields=__fields_mapper(fields)) + + +def fetch_dashboard_details(dashboard_id): + """Wrapper function to call REST and SDK fetch functions.""" + access_token = get_access_token() + fetch_dashboard(access_token, dashboard_id) + time.sleep(0.5) + fetch_dashboard_sdk(dashboard_id) + + +def main(): + setup_logging() + + check_environment_variables() + + selected_dashboard_ids = [ + "16651", + "17435", + "18347", + "20756", + "23242", + "24753", + "26597", + "27111", + "27760", + "20611", + "22974", + "23970", + "24882", + "29439", + "29673", + "29746", + "29862", + "32222", + "33057", + ] + + # selected_dashboard_ids: List[Optional[str]] = [] + dashboards = all_dashboards(fields="id") + # filtered_dashboards = [ + # "14740", + # "18966", + # "20900", + # "23479", + # "24614", + # "29246", + # "30699", + # "32630", + # "33846", + # ] + dashboard_ids = [dashboard_base.id for dashboard_base in dashboards] + logging.info(f"Total Dashboard {len(dashboard_ids)}") + # for id in dashboard_ids: + # if id not in filtered_dashboards: + # selected_dashboard_ids.append(id) + + logging.info(f"Seleted Dashboard {len(selected_dashboard_ids)}") + + with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + futures = [ + executor.submit(fetch_dashboard_details, dashboard_id) + for dashboard_id in selected_dashboard_ids + ] + for future in as_completed(futures): + try: + future.result() # This will raise any exceptions caught during the execution of the worker + except Exception as e: + logging.error(f"An error occurred: {e}") + + logging.info(f" Failed dashboard via rest {failed_dashboad_id_via_rest}") + logging.info(f" Failed dashboard via sdk {failed_dashboad_id_via_sdk}") + + +if __name__ == "__main__": + main() diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index 7ffb895349ed2..ea4853da09387 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -165,6 +165,15 @@ class LookMLSourceConfig( "view.sql_table_name. Defaults to an empty dictionary.", ) + lookml_parameter: Dict[Any, Any] = Field( + {}, + description=( + "A dictionary containing LookML parameters (`@{param_name}`) and their values. " + "Parameters are used to define dynamic references in LookML views, such as `view.sql_table_name`." + "Defaults to an empty dictionary." + ), + ) + looker_environment: Literal["prod", "dev"] = Field( "prod", description="A looker prod or dev environment. " From 36f746f48ad195cb418768d8c5aeb1a089d6152f Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 8 Jan 2025 16:43:25 +0530 Subject: [PATCH 03/13] test: updated golden files --- metadata-ingestion/tests/integration/lookml/test_lookml.py | 5 ++++- .../star_award_winner.view.lkml | 3 ++- .../lookml/vv_lineage_liquid_template_golden.json | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 66b49cdb6eaaa..6f1fed8ab1ac6 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -873,7 +873,10 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t "_is_selected": True, }, "source_region": "ap-south-1", - "star_award_winner_year": "public.winner_2025", + "db": "test-db", + } + new_recipe["source"]["config"]["lookml_parameter"] = { + "star_award_winner_year": "public.winner_2025" } pipeline = Pipeline.create(new_recipe) diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml index e8f1a8aa48f95..e8170feded72b 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml @@ -1,5 +1,6 @@ view: star_award_winner { - sql_table_name: @{star_award_winner_year} ;; + sql_table_name: ${db}.@{star_award_winner_year};; + dimension: id { label: "id" diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json index 9af13f2a6b0a1..596e20d9bbaf4 100644 --- a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -2853,7 +2853,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "view: star_award_winner {\n sql_table_name: @{star_award_winner_year} ;;\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n parameter: star_award_winner_year {\n type: string\n allowed_value: {\n label: \"Star Award Winner Of 2025\"\n value: \"public.winner_2025\"\n }\n allowed_value: {\n label: \"Star Award Winner Of 2024\"\n value: \"public.winner_2024\"\n }\n }\n\n}", + "viewLogic": "view: star_award_winner {\n sql_table_name: ${db}.@{star_award_winner_year};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n parameter: star_award_winner_year {\n type: string\n allowed_value: {\n label: \"Star Award Winner Of 2025\"\n value: \"public.winner_2025\"\n }\n allowed_value: {\n label: \"Star Award Winner Of 2024\"\n value: \"public.winner_2024\"\n }\n }\n\n}", "viewLanguage": "lookml" } }, @@ -2904,7 +2904,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,test-db.public.winner_2025,PROD)", "type": "VIEW" } ], @@ -2912,7 +2912,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test-db.public.winner_2025,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ From 4a9ec8853295a2d0e46cb14c0fb17f9397b4fb50 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 8 Jan 2025 16:47:34 +0530 Subject: [PATCH 04/13] fix: minor change --- .../source/looker/looker_template_language.py | 4 +- .../ingestion/source/looker/looker_test.py | 215 ------------------ 2 files changed, 1 insertion(+), 218 deletions(-) delete mode 100644 metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index bd7fef8fee189..d93fd6ea61070 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -362,7 +362,6 @@ def replace_parameters(match): if key in self.source_config.lookml_parameter: return str(self.source_config.lookml_parameter.get(key)) - # Log warning for missing parameter logger.warning(f"Parameter '@{{{key}}}' not found in configuration.") return "" @@ -372,7 +371,6 @@ def replace_liquid_variables(match): if key in self.source_config.liquid_variable: return str(self.source_config.liquid_variable.get(key)) - # Log warning for missing liquid variable logger.warning(f"Liquid variable '${{{key}}}' not found in configuration.") return "" @@ -455,7 +453,7 @@ def process_lookml_template_language( ), # Now resolve liquid variables LookmlParameterTransformer( source_config=source_config - ), # Remove @{variable} with its corresponding value + ), # Remove @{param}/${var} with its corresponding value DropDerivedViewPatternTransformer( source_config=source_config ), # Remove any ${} symbol diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py deleted file mode 100644 index f9e30d5323d47..0000000000000 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_test.py +++ /dev/null @@ -1,215 +0,0 @@ -import json -import logging -import os -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import List, Sequence, Union - -import looker_sdk -import requests -from looker_sdk.sdk.api40.models import DashboardBase - -failed_dashboad_id_via_rest = {} -failed_dashboad_id_via_sdk = {} - - -def setup_logging(): - """Set up dual logging to console and file.""" - # Create a logger - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - - # Create handlers for both file and console - file_handler = logging.FileHandler("looker_api_logs.log") - console_handler = logging.StreamHandler() - - # Set logging level for both handlers - file_handler.setLevel(logging.DEBUG) - console_handler.setLevel(logging.DEBUG) - - # Create a logging format - formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - - # Add handlers to the logger - logger.addHandler(file_handler) - logger.addHandler(console_handler) - - -def get_access_token(): - """Authenticate with Looker API and return the access token.""" - url = f"{os.getenv('LOOKERSDK_BASE_URL')}/api/4.0/login" - data = { - "client_id": os.getenv("LOOKERSDK_CLIENT_ID"), - "client_secret": os.getenv("LOOKERSDK_CLIENT_SECRET"), - } - response = requests.post(url, data=data) - if response.status_code == 200: - return response.json()["access_token"] - else: - logging.error("Failed to authenticate with Looker API") - logging.error(response.text) - raise Exception("Authentication failed") - - -def __fields_mapper(fields: Union[str, List[str]]) -> str: - """Helper method to turn single string or list of fields into Looker API compatible fields param""" - return fields if isinstance(fields, str) else ",".join(fields) - - -def fetch_dashboard_sdk(dashboard_id): - sdk = looker_sdk.init40() - logging.info( - f"---------------------- Fetching Dashboard ID {dashboard_id} Via SDK ----------------------" - ) - try: - fields = [ - "id", - "title", - "dashboard_elements", - "dashboard_filters", - "deleted", - "hidden", - "description", - "folder", - "user_id", - "created_at", - "updated_at", - "last_updater_id", - "deleted_at", - "deleter_id", - ] - dashboard = sdk.dashboard(dashboard_id, fields=__fields_mapper(fields)) - logging.info( - f"Dashboard ID {dashboard_id} details fetched successfully via SDK:" - ) - logging.info(json.dumps(dashboard, indent=2, default=str)) - except Exception as e: - logging.error(f"SDK Error for Dashboard ID {dashboard_id}: {e}") - failed_dashboad_id_via_sdk[dashboard_id] = str(e) - logging.info( - f"---------------------- Request Completed {dashboard_id} Via SDK----------------------" - ) - - -def fetch_dashboard(access_token, dashboard_id): - """Fetch a dashboard by ID using the Looker API and log with separators.""" - headers = { - "Authorization": f"token {access_token}", - "Content-Type": "application/json", - } - url = f"{os.getenv('LOOKERSDK_BASE_URL')}/api/4.0/dashboards/{dashboard_id}" - response = requests.get(url, headers=headers) - - logging.info( - f"---------------------- Fetching Dashboard ID {dashboard_id} ----------------------" - ) - - if response.status_code == 200: - logging.info(f"Dashboard ID {dashboard_id} details fetched successfully:") - logging.info(json.dumps(response.json(), indent=2)) - else: - logging.error(f"Failed to fetch Dashboard ID {dashboard_id}:") - value = response.content.decode(encoding="utf-8") - logging.error(value) - failed_dashboad_id_via_rest[dashboard_id] = response.text - logging.info( - f"---------------------- Request Completed {dashboard_id} ----------------------" - ) - - -def check_environment_variables(): - """Check if all required environment variables are set and log which are missing.""" - required_vars = [ - "LOOKERSDK_CLIENT_ID", - "LOOKERSDK_CLIENT_SECRET", - "LOOKERSDK_BASE_URL", - ] - missing_vars = [var for var in required_vars if not os.getenv(var)] - if missing_vars: - error_message = ( - f"Missing required environment variables: {', '.join(missing_vars)}" - ) - logging.error(error_message) - raise EnvironmentError(error_message) - - -def all_dashboards(fields: Union[str, List[str]]) -> Sequence[DashboardBase]: - sdk = looker_sdk.init40() - return sdk.all_dashboards(fields=__fields_mapper(fields)) - - -def fetch_dashboard_details(dashboard_id): - """Wrapper function to call REST and SDK fetch functions.""" - access_token = get_access_token() - fetch_dashboard(access_token, dashboard_id) - time.sleep(0.5) - fetch_dashboard_sdk(dashboard_id) - - -def main(): - setup_logging() - - check_environment_variables() - - selected_dashboard_ids = [ - "16651", - "17435", - "18347", - "20756", - "23242", - "24753", - "26597", - "27111", - "27760", - "20611", - "22974", - "23970", - "24882", - "29439", - "29673", - "29746", - "29862", - "32222", - "33057", - ] - - # selected_dashboard_ids: List[Optional[str]] = [] - dashboards = all_dashboards(fields="id") - # filtered_dashboards = [ - # "14740", - # "18966", - # "20900", - # "23479", - # "24614", - # "29246", - # "30699", - # "32630", - # "33846", - # ] - dashboard_ids = [dashboard_base.id for dashboard_base in dashboards] - logging.info(f"Total Dashboard {len(dashboard_ids)}") - # for id in dashboard_ids: - # if id not in filtered_dashboards: - # selected_dashboard_ids.append(id) - - logging.info(f"Seleted Dashboard {len(selected_dashboard_ids)}") - - with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: - futures = [ - executor.submit(fetch_dashboard_details, dashboard_id) - for dashboard_id in selected_dashboard_ids - ] - for future in as_completed(futures): - try: - future.result() # This will raise any exceptions caught during the execution of the worker - except Exception as e: - logging.error(f"An error occurred: {e}") - - logging.info(f" Failed dashboard via rest {failed_dashboad_id_via_rest}") - logging.info(f" Failed dashboard via sdk {failed_dashboad_id_via_sdk}") - - -if __name__ == "__main__": - main() From 750cf33dcb760fdb1e02a41b2b7434e56f42556c Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 8 Jan 2025 22:04:09 +0530 Subject: [PATCH 05/13] fix: pr comments --- .../src/datahub/ingestion/source/looker/lookml_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index ea4853da09387..4092903d04f0d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -165,7 +165,7 @@ class LookMLSourceConfig( "view.sql_table_name. Defaults to an empty dictionary.", ) - lookml_parameter: Dict[Any, Any] = Field( + lookml_parameter: Dict[str, str] = Field( {}, description=( "A dictionary containing LookML parameters (`@{param_name}`) and their values. " From 4e7bc947a7365a671b736270b7f062b9a5c9c363 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Fri, 10 Jan 2025 11:21:31 +0530 Subject: [PATCH 06/13] fix: pr comment --- .../docs/sources/looker/looker_recipe.yml | 15 ++++ .../docs/sources/looker/lookml_post.md | 87 +++++++++++++++++-- .../source/looker/looker_template_language.py | 10 +-- .../ingestion/source/looker/lookml_config.py | 4 +- 4 files changed, 104 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/docs/sources/looker/looker_recipe.yml b/metadata-ingestion/docs/sources/looker/looker_recipe.yml index 42209f8cc6809..eb1ed06f70d62 100644 --- a/metadata-ingestion/docs/sources/looker/looker_recipe.yml +++ b/metadata-ingestion/docs/sources/looker/looker_recipe.yml @@ -8,4 +8,19 @@ source: client_id: ${LOOKER_CLIENT_ID} client_secret: ${LOOKER_CLIENT_SECRET} + # Liquid variables + # liquid_variable: + # _user_attributes: + # looker_env: "dev" + # dev_database_prefix: "employee" + # dev_schema_prefix: "public" + # dw_eff_dt_date: + # _is_selected: true + # source_region: "ap-south-1" + # db: "test-db" + + # LookML parameters + # lookml_parameter: + # star_award_winner_year: "public.winner_2025" + # sink configs diff --git a/metadata-ingestion/docs/sources/looker/lookml_post.md b/metadata-ingestion/docs/sources/looker/lookml_post.md index 8a4bf823ffc27..5fbabd0c0fdad 100644 --- a/metadata-ingestion/docs/sources/looker/lookml_post.md +++ b/metadata-ingestion/docs/sources/looker/lookml_post.md @@ -1,11 +1,88 @@ #### Configuration Notes 1. If a view contains a liquid template (e.g. `sql_table_name: {{ user_attributes['db']}}.kafka_streaming.events }}`, with `db=ANALYTICS_PROD`), then you will need to specify the values of those variables in the `liquid_variable` config as shown below: - ```yml - liquid_variable: - user_attributes: - db: ANALYTICS_PROD - ``` + + ```yml + liquid_variable: + user_attributes: + db: ANALYTICS_PROD + ``` + +2. If a view contains a liquid variable (e.g., `sql_table_name: ${db}.kafka_streaming.events;`), with `db=ANALYTICS_PROD`, you will need to specify the value of the variable in the `liquid_variable` config as shown below: + + ```yml + liquid_variable: + db: ANALYTICS_PROD + ``` + + **Example**: + + ```lkml + sql_table_name: ${db}.kafka_streaming.events;; + ``` + + **Resolved Output**: + + ``` + ANALYTICS_PROD.kafka_streaming.events + ``` + +3. If a view contains a LookML parameter (e.g., `sql_table_name: @{year}.public.events;`), with `year=public.winner_2025`, you will need to specify the value of the parameter in the `lookml_parameter` config as shown below: + + ```yml + lookml_parameter: + year: public.winner_2025 + ``` + + **Example**: + + ```lkml + parameter: year { + type: string + allowed_value: { + label: "2025" + value: "public.winner_2025" + } + } + + sql_table_name: @{year}.public.events;; + ``` + + **Resolved Output**: + + ``` + public.winner_2025.public.events + ``` + +4. If a view contains both liquid variables and LookML parameters (e.g., `sql_table_name: ${db}.@{year}.events;`), you will need to configure both in your recipe as shown below: + + ```yml + liquid_variable: + db: ANALYTICS_PROD + + lookml_parameter: + year: public.winner_2025 + ``` + + **Example**: + + ```lkml + parameter: year { + type: string + allowed_value: { + label: "2025" + value: "public.winner_2025" + } + } + + sql_table_name: ${db}.@{year}.events;; + ``` + + **Resolved Output**: + + ``` + ANALYTICS_PROD.public.winner_2025.events + ``` ### Multi-Project LookML (Advanced) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index d93fd6ea61070..9d73bff4e5c0b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -351,6 +351,10 @@ def resolve_lookml_parameter(self, text: str) -> str: def replace_parameters(match): key = match.group(1) + # Resolve parameter + if key in self.source_config.lookml_parameter: + return str(self.source_config.lookml_parameter.get(key)) + # Check if it's a misplaced liquid variable if key in self.source_config.liquid_variable: logger.warning( @@ -358,10 +362,6 @@ def replace_parameters(match): ) return f"@{{{key}}}" - # Resolve parameter - if key in self.source_config.lookml_parameter: - return str(self.source_config.lookml_parameter.get(key)) - logger.warning(f"Parameter '@{{{key}}}' not found in configuration.") return "" @@ -453,7 +453,7 @@ def process_lookml_template_language( ), # Now resolve liquid variables LookmlParameterTransformer( source_config=source_config - ), # Remove @{param}/${var} with its corresponding value + ), # Resolve @{param} with its corresponding value DropDerivedViewPatternTransformer( source_config=source_config ), # Remove any ${} symbol diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index 4092903d04f0d..ee5e2820d7340 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -161,8 +161,8 @@ class LookMLSourceConfig( liquid_variable: Dict[Any, Any] = Field( {}, description="A dictionary containing Liquid variables and their corresponding values, utilized in SQL-defined " - "derived views. The Liquid template will be resolved in view.derived_table.sql and " - "view.sql_table_name. Defaults to an empty dictionary.", + "derived views. The Liquid template will be resolved in `view.derived_table.sql` and " + "`view.sql_table_name`. Variables use the `${var}` syntax for substitution. Defaults to an empty dictionary.", ) lookml_parameter: Dict[str, str] = Field( From ae605d966498f465b6eb1324afa7e6a046d9d571 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 15 Jan 2025 16:09:11 +0530 Subject: [PATCH 07/13] fix: refactor changes to lookml constant --- .../docs/sources/looker/looker_recipe.yml | 4 +- .../docs/sources/looker/lookml_post.md | 63 +----------------- .../source/looker/looker_file_loader.py | 5 +- .../source/looker/looker_template_language.py | 64 ++++++++++--------- .../ingestion/source/looker/lookml_config.py | 10 +-- .../ingestion/source/looker/lookml_source.py | 28 ++++++-- 6 files changed, 70 insertions(+), 104 deletions(-) diff --git a/metadata-ingestion/docs/sources/looker/looker_recipe.yml b/metadata-ingestion/docs/sources/looker/looker_recipe.yml index eb1ed06f70d62..346005e621aff 100644 --- a/metadata-ingestion/docs/sources/looker/looker_recipe.yml +++ b/metadata-ingestion/docs/sources/looker/looker_recipe.yml @@ -19,8 +19,8 @@ source: # source_region: "ap-south-1" # db: "test-db" - # LookML parameters - # lookml_parameter: + # LookML Constant + # lookml_constant: # star_award_winner_year: "public.winner_2025" # sink configs diff --git a/metadata-ingestion/docs/sources/looker/lookml_post.md b/metadata-ingestion/docs/sources/looker/lookml_post.md index 5fbabd0c0fdad..175980c24c3d8 100644 --- a/metadata-ingestion/docs/sources/looker/lookml_post.md +++ b/metadata-ingestion/docs/sources/looker/lookml_post.md @@ -8,17 +8,17 @@ db: ANALYTICS_PROD ``` -2. If a view contains a liquid variable (e.g., `sql_table_name: ${db}.kafka_streaming.events;`), with `db=ANALYTICS_PROD`, you will need to specify the value of the variable in the `liquid_variable` config as shown below: +2. If a view contains a LookML constant (e.g., `sql_table_name: @{db}.kafka_streaming.events;`), its value will be resolved by first checking the manifest file and, if not found, then checking the config as shown below:. ```yml - liquid_variable: + lookml_constant: db: ANALYTICS_PROD ``` **Example**: ```lkml - sql_table_name: ${db}.kafka_streaming.events;; + sql_table_name: @{db}.kafka_streaming.events;; ``` **Resolved Output**: @@ -27,63 +27,6 @@ ANALYTICS_PROD.kafka_streaming.events ``` -3. If a view contains a LookML parameter (e.g., `sql_table_name: @{year}.public.events;`), with `year=public.winner_2025`, you will need to specify the value of the parameter in the `lookml_parameter` config as shown below: - - ```yml - lookml_parameter: - year: public.winner_2025 - ``` - - **Example**: - - ```lkml - parameter: year { - type: string - allowed_value: { - label: "2025" - value: "public.winner_2025" - } - } - - sql_table_name: @{year}.public.events;; - ``` - - **Resolved Output**: - - ``` - public.winner_2025.public.events - ``` - -4. If a view contains both liquid variables and LookML parameters (e.g., `sql_table_name: ${db}.@{year}.events;`), you will need to configure both in your recipe as shown below: - - ```yml - liquid_variable: - db: ANALYTICS_PROD - - lookml_parameter: - year: public.winner_2025 - ``` - - **Example**: - - ```lkml - parameter: year { - type: string - allowed_value: { - label: "2025" - value: "public.winner_2025" - } - } - - sql_table_name: ${db}.@{year}.events;; - ``` - - **Resolved Output**: - - ``` - ANALYTICS_PROD.public.winner_2025.events - ``` - ### Multi-Project LookML (Advanced) Looker projects support organization as multiple git repos, with [remote includes that can refer to projects that are stored in a different repo](https://cloud.google.com/looker/docs/importing-projects#include_files_from_an_imported_project). If your Looker implementation uses multi-project setup, you can configure the LookML source to pull in metadata from your remote projects as well. diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py index 9fac0b52fde0d..e9c996d590248 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py @@ -1,7 +1,7 @@ import logging import pathlib from dataclasses import replace -from typing import Dict, Optional +from typing import Dict, List, Optional from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition from datahub.ingestion.source.looker.looker_dataclasses import LookerViewFile @@ -30,12 +30,14 @@ def __init__( base_projects_folder: Dict[str, pathlib.Path], reporter: LookMLSourceReport, source_config: LookMLSourceConfig, + looker_constant: Optional[List[Dict[str, str]]] = [], ) -> None: self.viewfile_cache: Dict[str, Optional[LookerViewFile]] = {} self._root_project_name = root_project_name self._base_projects_folder = base_projects_folder self.reporter = reporter self.source_config = source_config + self.looker_constant = looker_constant def _load_viewfile( self, project_name: str, path: str, reporter: LookMLSourceReport @@ -74,6 +76,7 @@ def _load_viewfile( parsed = load_and_preprocess_file( path=path, source_config=self.source_config, + looker_constant=self.looker_constant, ) looker_viewfile = LookerViewFile.from_looker_dict( diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index 9d73bff4e5c0b..7787567d3736e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -192,8 +192,13 @@ class LookMLViewTransformer(ABC): source_config: LookMLSourceConfig - def __init__(self, source_config: LookMLSourceConfig): + def __init__( + self, + source_config: LookMLSourceConfig, + looker_constant: Optional[List[Dict[str, str]]] = [], + ): self.source_config = source_config + self.looker_constant = looker_constant def transform(self, view: dict) -> dict: value_to_transform: Optional[str] = None @@ -335,25 +340,36 @@ def _apply_transformation(self, value: str, view: dict) -> str: return self._apply_regx(value) -class LookmlParameterTransformer(LookMLViewTransformer): +class LookmlConstantTransformer(LookMLViewTransformer): """ - Replace LookML parameters (@{param} or ${var}) from the configuration. + Replace LookML constants @{constant} from the manifest/configuration. """ - PARAMETER_PATTERN = r"@{(\w+)}" # Matches @{param} - LIQUID_VARIABLE_PATTERN = r"\${(\w+)}" # Matches ${var} + CONSTANT_PATTERN = r"@{(\w+)}" # Matches @{constant} def resolve_lookml_parameter(self, text: str) -> str: """ - Resolves LookML parameters (@{param}) and liquid variables (${var}). + Resolves LookML constants (@{ }) from manifest or config. Logs warnings for misplaced or missing variables. """ - def replace_parameters(match): + def replace_constants(match): key = match.group(1) - # Resolve parameter - if key in self.source_config.lookml_parameter: - return str(self.source_config.lookml_parameter.get(key)) + if self.looker_constant: + value = next( + ( + item["value"] + for item in self.looker_constant + if item["name"] == key + ), + None, + ) + if value: + return value + + # Resolve constant + if key in self.source_config.lookml_constant: + return str(self.source_config.lookml_constant.get(key)) # Check if it's a misplaced liquid variable if key in self.source_config.liquid_variable: @@ -363,23 +379,10 @@ def replace_parameters(match): return f"@{{{key}}}" logger.warning(f"Parameter '@{{{key}}}' not found in configuration.") - return "" + return "NULL" - def replace_liquid_variables(match): - key = match.group(1) - # Resolve liquid variable - if key in self.source_config.liquid_variable: - return str(self.source_config.liquid_variable.get(key)) - - logger.warning(f"Liquid variable '${{{key}}}' not found in configuration.") - return "" - - # Resolve @{param} (parameters) - text = re.sub(self.PARAMETER_PATTERN, replace_parameters, text) - - # Resolve ${var} (liquid variables) - text = re.sub(self.LIQUID_VARIABLE_PATTERN, replace_liquid_variables, text) - return text + # Resolve @{} (constant) + return re.sub(self.CONSTANT_PATTERN, replace_constants, text) def _apply_transformation(self, value: str, view: dict) -> str: return self.resolve_lookml_parameter(text=value) @@ -440,6 +443,7 @@ def view(self) -> dict: def process_lookml_template_language( source_config: LookMLSourceConfig, view_lkml_file_dict: dict, + looker_constant: Optional[List[Dict[str, str]]], ) -> None: if "views" not in view_lkml_file_dict: return @@ -451,9 +455,9 @@ def process_lookml_template_language( LiquidVariableTransformer( source_config=source_config ), # Now resolve liquid variables - LookmlParameterTransformer( - source_config=source_config - ), # Resolve @{param} with its corresponding value + LookmlConstantTransformer( + source_config=source_config, looker_constant=looker_constant + ), # Resolve @{} constant with its corresponding value DropDerivedViewPatternTransformer( source_config=source_config ), # Remove any ${} symbol @@ -475,12 +479,14 @@ def process_lookml_template_language( def load_and_preprocess_file( path: Union[str, pathlib.Path], source_config: LookMLSourceConfig, + looker_constant: Optional[List[Dict[str, str]]] = [], ) -> dict: parsed = load_lkml(path) process_lookml_template_language( view_lkml_file_dict=parsed, source_config=source_config, + looker_constant=looker_constant, ) return parsed diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index ee5e2820d7340..fac411ac75e05 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -160,15 +160,15 @@ class LookMLSourceConfig( liquid_variable: Dict[Any, Any] = Field( {}, - description="A dictionary containing Liquid variables and their corresponding values, utilized in SQL-defined " - "derived views. The Liquid template will be resolved in `view.derived_table.sql` and " - "`view.sql_table_name`. Variables use the `${var}` syntax for substitution. Defaults to an empty dictionary.", + description="A dictionary containing Liquid variables, Liquid logic, and LookML parameters with their corresponding values, utilized in SQL-defined " + "derived views. The Liquid template will be resolved in view.derived_table.sql and " + "view.sql_table_name. Defaults to an empty dictionary.", ) - lookml_parameter: Dict[str, str] = Field( + lookml_constant: Dict[str, str] = Field( {}, description=( - "A dictionary containing LookML parameters (`@{param_name}`) and their values. " + "A dictionary containing LookML constant (`@{constant_name}`) and their values. " "Parameters are used to define dynamic references in LookML views, such as `view.sql_table_name`." "Defaults to an empty dictionary." ), diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index c7d3724472d3c..1a9bc611cbf7f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -253,6 +253,7 @@ class LookerManifest: # This must be set if the manifest has local_dependency entries. # See https://cloud.google.com/looker/docs/reference/param-manifest-project-name project_name: Optional[str] + constants: Optional[List[Dict[str, str]]] local_dependencies: List[str] remote_dependencies: List[LookerRemoteDependency] @@ -506,6 +507,7 @@ def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManife manifest = LookerManifest( project_name=manifest_dict.get("project_name"), + constants=manifest_dict.get("constants", []), local_dependencies=[ x["project"] for x in manifest_dict.get("local_dependencys", []) ], @@ -546,6 +548,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ] = self.source_config.base_folder visited_projects: Set[str] = set() + looker_constant: List[Dict[str, str]] = [] # We clone everything that we're pointed at. for project, p_ref in self.source_config.project_dependencies.items(): @@ -574,10 +577,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.base_projects_folder[project] = p_ref self._recursively_check_manifests( - tmp_dir, BASE_PROJECT_NAME, visited_projects + tmp_dir, BASE_PROJECT_NAME, visited_projects, looker_constant ) - yield from self.get_internal_workunits() + yield from self.get_internal_workunits(looker_constant) if not self.report.events_produced and not self.report.failures: # Don't pass if we didn't produce any events. @@ -587,7 +590,11 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) def _recursively_check_manifests( - self, tmp_dir: str, project_name: str, project_visited: Set[str] + self, + tmp_dir: str, + project_name: str, + project_visited: Set[str], + looker_constant: List[Dict[str, str]], ) -> None: if project_name in project_visited: return @@ -604,6 +611,9 @@ def _recursively_check_manifests( if not manifest: return + if manifest.constants: + looker_constant.extend(manifest.constants) + # Special case handling if the root project has a name in the manifest file. if project_name == BASE_PROJECT_NAME and manifest.project_name: if ( @@ -663,20 +673,24 @@ def _recursively_check_manifests( project_visited.add(project_name) else: self._recursively_check_manifests( - tmp_dir, remote_project.name, project_visited + tmp_dir, remote_project.name, project_visited, looker_constant ) for project in manifest.local_dependencies: - self._recursively_check_manifests(tmp_dir, project, project_visited) + self._recursively_check_manifests( + tmp_dir, project, project_visited, looker_constant + ) - def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 + def get_internal_workunits( + self, looker_constant: Optional[List[Dict]] + ) -> Iterable[MetadataWorkUnit]: # noqa: C901 assert self.source_config.base_folder - viewfile_loader = LookerViewFileLoader( self.source_config.project_name, self.base_projects_folder, self.reporter, self.source_config, + looker_constant, ) # Some views can be mentioned by multiple 'include' statements and can be included via different connections. From a7cc9d5b0a73b90c723fe69528e8e2170be0a2fc Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 15 Jan 2025 16:11:03 +0530 Subject: [PATCH 08/13] test: updated test for lookml constant --- .../tests/integration/lookml/test_lookml.py | 8 +- .../activity_logs.view.lkml | 9 +- .../data.model.lkml | 7 +- .../manifest.lkml | 15 ++ .../star_award_winner.view.lkml | 14 +- .../star_award_winner_dev.view.lkml | 17 ++ .../vv_lineage_liquid_template_golden.json | 230 +++++++++++++++++- 7 files changed, 268 insertions(+), 32 deletions(-) create mode 100644 metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/manifest.lkml create mode 100644 metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner_dev.view.lkml diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 6f1fed8ab1ac6..532984f6da0d7 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -869,15 +869,15 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t "dev_database_prefix": "employee", "dev_schema_prefix": "public", }, + "_dev_database_attributes": { + "dev_database_prefix": "customer", + }, "dw_eff_dt_date": { "_is_selected": True, }, "source_region": "ap-south-1", - "db": "test-db", - } - new_recipe["source"]["config"]["lookml_parameter"] = { - "star_award_winner_year": "public.winner_2025" } + new_recipe["source"]["config"]["lookml_constant"] = {"winner_table": "dev"} pipeline = Pipeline.create(new_recipe) pipeline.run() diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml index f0e2dec6e4678..8512e1bf0469b 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml @@ -1,13 +1,6 @@ view: activity_logs { sql_table_name: - {% if _user_attributes['looker_env'] == 'dev' %} - {{ _user_attributes['dev_database_prefix'] }}analytics.{{ _user_attributes['dev_schema_prefix'] }}staging_app.stg_app__activity_logs - {% elsif _user_attributes['looker_env'] == 'prod' %} - analytics.staging_app.stg_app__activity_logs - {% else %} - analytics.staging_app.stg_app__activity_logs - {% endif %} - ;; + {{ _dev_database_attributes['dev_database_prefix'] }}_analytics.{{ _user_attributes['dev_schema_prefix'] }}_staging_app.stg_app__activity_logs;; dimension: generated_message_id { group_label: "IDs" diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml index ae6f70443ab56..1028129a14fc5 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml @@ -11,6 +11,8 @@ include: "employee_income_source_as_per_env.view.lkml" include: "rent_as_employee_income_source.view.lkml" include: "child_view.view.lkml" include: "star_award_winner.view.lkml" +include: "star_award_winner_dev.view.lkml" + explore: activity_logs { } @@ -43,4 +45,7 @@ explore: child_view { } explore: star_award_winner { -} \ No newline at end of file +} + +explore: star_award_winner_dev { +} diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/manifest.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/manifest.lkml new file mode 100644 index 0000000000000..fcdd71a626294 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/manifest.lkml @@ -0,0 +1,15 @@ +constant: customer_support_db { + value: "star_award_winner_year" + export: none +} + +constant: customer_support_schema { + value: "public" + export: none +} + +constant: customer_support_table { + value: "winner" + export: none +} + diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml index e8170feded72b..64999cac98bb2 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml @@ -1,5 +1,5 @@ view: star_award_winner { - sql_table_name: ${db}.@{star_award_winner_year};; + sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{customer_support_table};; dimension: id { @@ -9,16 +9,4 @@ view: star_award_winner { sql: ${TABLE}.id ;; } - parameter: star_award_winner_year { - type: string - allowed_value: { - label: "Star Award Winner Of 2025" - value: "public.winner_2025" - } - allowed_value: { - label: "Star Award Winner Of 2024" - value: "public.winner_2024" - } - } - } \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner_dev.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner_dev.view.lkml new file mode 100644 index 0000000000000..0c2417251fc15 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner_dev.view.lkml @@ -0,0 +1,17 @@ +view: star_award_winner_dev { + sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{winner_table};; + + + dimension: id { + label: "id" + primary_key: yes + type: number + sql: ${TABLE}.id ;; + } + + dimension: name { + type: string + sql: ${TABLE}.name;; + } + +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json index 596e20d9bbaf4..1938c029bc924 100644 --- a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -117,7 +117,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "view: activity_logs {\n sql_table_name:\n {% if _user_attributes['looker_env'] == 'dev' %}\n {{ _user_attributes['dev_database_prefix'] }}analytics.{{ _user_attributes['dev_schema_prefix'] }}staging_app.stg_app__activity_logs\n {% elsif _user_attributes['looker_env'] == 'prod' %}\n analytics.staging_app.stg_app__activity_logs\n {% else %}\n analytics.staging_app.stg_app__activity_logs\n {% endif %}\n ;;\n\n dimension: generated_message_id {\n group_label: \"IDs\"\n primary_key: yes\n type: number\n sql: ${TABLE}.\"GENERATED_MESSAGE_ID\" ;;\n }\n}\n", + "viewLogic": "view: activity_logs {\n sql_table_name:\n {{ _dev_database_attributes['dev_database_prefix'] }}_analytics.{{ _user_attributes['dev_schema_prefix'] }}_staging_app.stg_app__activity_logs;;\n\n dimension: generated_message_id {\n group_label: \"IDs\"\n primary_key: yes\n type: number\n sql: ${TABLE}.\"GENERATED_MESSAGE_ID\" ;;\n }\n}\n", "viewLanguage": "lookml" } }, @@ -168,7 +168,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,employeeanalytics.publicstaging_app.stg_app__activity_logs,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,customer_analytics.public_staging_app.stg_app__activity_logs,PROD)", "type": "VIEW" } ], @@ -176,7 +176,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,employeeanalytics.publicstaging_app.stg_app__activity_logs,PROD),generated_message_id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,customer_analytics.public_staging_app.stg_app__activity_logs,PROD),generated_message_id)" ], "downstreamType": "FIELD", "downstreams": [ @@ -2853,7 +2853,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "view: star_award_winner {\n sql_table_name: ${db}.@{star_award_winner_year};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n parameter: star_award_winner_year {\n type: string\n allowed_value: {\n label: \"Star Award Winner Of 2025\"\n value: \"public.winner_2025\"\n }\n allowed_value: {\n label: \"Star Award Winner Of 2024\"\n value: \"public.winner_2024\"\n }\n }\n\n}", + "viewLogic": "view: star_award_winner {\n sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{customer_support_table};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n}", "viewLanguage": "lookml" } }, @@ -2904,7 +2904,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,test-db.public.winner_2025,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.winner,PROD)", "type": "VIEW" } ], @@ -2912,7 +2912,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test-db.public.winner_2025,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.winner,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ @@ -3013,6 +3013,224 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: star_award_winner_dev {\n sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{winner_table};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n dimension: name {\n type: string\n sql: ${TABLE}.name;;\n }\n\n}", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD),name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "star_award_winner_dev", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "id", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": true + }, + { + "fieldPath": "name", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [ + "id" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "star_award_winner_dev.view.lkml", + "looker.model": "data" + }, + "name": "star_award_winner_dev", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", From 46316ca339c7e81176a8b37ea6d238e4b067acae Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Wed, 15 Jan 2025 16:58:23 +0530 Subject: [PATCH 09/13] fix: fixed lint issue --- .../ingestion/source/looker/lookml_source.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 1a9bc611cbf7f..68775d28bc453 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -310,6 +310,8 @@ def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext): "manage_models permission enabled on this API key." ) from err + self.looker_constant: List[Dict[str, str]] = [] + def _load_model(self, path: str) -> LookerModel: logger.debug(f"Loading model from file {path}") @@ -548,7 +550,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ] = self.source_config.base_folder visited_projects: Set[str] = set() - looker_constant: List[Dict[str, str]] = [] # We clone everything that we're pointed at. for project, p_ref in self.source_config.project_dependencies.items(): @@ -577,10 +578,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.base_projects_folder[project] = p_ref self._recursively_check_manifests( - tmp_dir, BASE_PROJECT_NAME, visited_projects, looker_constant + tmp_dir, BASE_PROJECT_NAME, visited_projects, self.looker_constant ) - yield from self.get_internal_workunits(looker_constant) + yield from self.get_internal_workunits() if not self.report.events_produced and not self.report.failures: # Don't pass if we didn't produce any events. @@ -681,16 +682,14 @@ def _recursively_check_manifests( tmp_dir, project, project_visited, looker_constant ) - def get_internal_workunits( - self, looker_constant: Optional[List[Dict]] - ) -> Iterable[MetadataWorkUnit]: # noqa: C901 + def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 assert self.source_config.base_folder viewfile_loader = LookerViewFileLoader( self.source_config.project_name, self.base_projects_folder, self.reporter, self.source_config, - looker_constant, + self.looker_constant, ) # Some views can be mentioned by multiple 'include' statements and can be included via different connections. From 2c16fc66fb4560f67928adeca68da0ccdfbc6a5e Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Thu, 16 Jan 2025 15:23:30 +0530 Subject: [PATCH 10/13] fix: pr comments --- .../docs/sources/looker/looker_recipe.yml | 2 +- .../docs/sources/looker/lookml_post.md | 32 ++++----- .../source/looker/looker_dataclasses.py | 2 + .../source/looker/looker_file_loader.py | 7 +- .../source/looker/looker_template_language.py | 70 ++++++++++++------- .../ingestion/source/looker/lookml_config.py | 7 +- .../ingestion/source/looker/lookml_source.py | 25 ++++--- 7 files changed, 87 insertions(+), 58 deletions(-) diff --git a/metadata-ingestion/docs/sources/looker/looker_recipe.yml b/metadata-ingestion/docs/sources/looker/looker_recipe.yml index 346005e621aff..388824df20052 100644 --- a/metadata-ingestion/docs/sources/looker/looker_recipe.yml +++ b/metadata-ingestion/docs/sources/looker/looker_recipe.yml @@ -20,7 +20,7 @@ source: # db: "test-db" # LookML Constant - # lookml_constant: + # lookml_constants: # star_award_winner_year: "public.winner_2025" # sink configs diff --git a/metadata-ingestion/docs/sources/looker/lookml_post.md b/metadata-ingestion/docs/sources/looker/lookml_post.md index 175980c24c3d8..77893ff2f17b8 100644 --- a/metadata-ingestion/docs/sources/looker/lookml_post.md +++ b/metadata-ingestion/docs/sources/looker/lookml_post.md @@ -8,24 +8,20 @@ db: ANALYTICS_PROD ``` -2. If a view contains a LookML constant (e.g., `sql_table_name: @{db}.kafka_streaming.events;`), its value will be resolved by first checking the manifest file and, if not found, then checking the config as shown below:. - - ```yml - lookml_constant: - db: ANALYTICS_PROD - ``` - - **Example**: - - ```lkml - sql_table_name: @{db}.kafka_streaming.events;; - ``` - - **Resolved Output**: - - ``` - ANALYTICS_PROD.kafka_streaming.events - ``` +2. If a view contains a LookML constant (e.g., `sql_table_name: @{db}.kafka_streaming.events;`), its value is resolved in the following order: + +- **First, checks the `manifest.lkml` file** for the constant definition. + ```manifest.lkml + constant: db { + value: "ANALYTICS_PROD" + } + ``` +- **If not found, falls back to `config`** + + ```yml + lookml_constants: + db: ANALYTICS_PROD + ``` ### Multi-Project LookML (Advanced) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py index d771821a14d88..a2e0931ebe4ea 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py @@ -75,6 +75,7 @@ def from_looker_dict( try: parsed = load_and_preprocess_file( path=included_file, + reporter=reporter, source_config=source_config, ) included_explores = parsed.get("explores", []) @@ -217,6 +218,7 @@ def resolve_includes( try: parsed = load_and_preprocess_file( path=included_file, + reporter=reporter, source_config=source_config, ) seen_so_far.add(included_file) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py index e9c996d590248..501cf3829a0e5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py @@ -30,14 +30,14 @@ def __init__( base_projects_folder: Dict[str, pathlib.Path], reporter: LookMLSourceReport, source_config: LookMLSourceConfig, - looker_constant: Optional[List[Dict[str, str]]] = [], + manifest_lookml_constant: Optional[List[Dict[str, str]]] = [], ) -> None: self.viewfile_cache: Dict[str, Optional[LookerViewFile]] = {} self._root_project_name = root_project_name self._base_projects_folder = base_projects_folder self.reporter = reporter self.source_config = source_config - self.looker_constant = looker_constant + self.manifest_lookml_constant = manifest_lookml_constant def _load_viewfile( self, project_name: str, path: str, reporter: LookMLSourceReport @@ -75,8 +75,9 @@ def _load_viewfile( parsed = load_and_preprocess_file( path=path, + reporter=self.reporter, source_config=self.source_config, - looker_constant=self.looker_constant, + manifest_lookml_constant=self.manifest_lookml_constant, ) looker_viewfile = LookerViewFile.from_looker_dict( diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index 7787567d3736e..99ed2bd0aef18 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -27,6 +27,7 @@ from datahub.ingestion.source.looker.lookml_config import ( DERIVED_VIEW_PATTERN, LookMLSourceConfig, + LookMLSourceReport, ) logger = logging.getLogger(__name__) @@ -82,7 +83,9 @@ def liquid_variable_with_default(self, text: str) -> dict: return self._create_new_liquid_variables_with_default(variables=variables) -def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str: +def resolve_liquid_variable( + text: str, liquid_variable: Dict[Any, Any], report: LookMLSourceReport +) -> str: # Set variable value to NULL if not present in liquid_variable dictionary Undefined.__str__ = lambda instance: "NULL" # type: ignore try: @@ -96,15 +99,22 @@ def resolve_liquid_variable(text: str, liquid_variable: Dict[Any, Any]) -> str: # Resolve liquid template return create_template(text).render(liquid_variable) except LiquidSyntaxError as e: - logger.warning(f"Unsupported liquid template encountered. error [{e.message}]") + report.report_warning( + message="Unsupported liquid template", + context=f"text {text} liquid_variable {liquid_variable}", + exc=e, + ) # TODO: There are some tag specific to looker and python-liquid library does not understand them. currently # we are not parsing such liquid template. # # See doc: https://cloud.google.com/looker/docs/templated-filters and look for { % condition region %} # order.region { % endcondition %} except CustomTagException as e: - logger.warning(e) - logger.debug(e, exc_info=e) + report.warning( + message="Unsupported liquid template", + context=f"text {text} liquid_variable {liquid_variable}", + exc=e, + ) return text @@ -195,10 +205,12 @@ class LookMLViewTransformer(ABC): def __init__( self, source_config: LookMLSourceConfig, - looker_constant: Optional[List[Dict[str, str]]] = [], + reporter: LookMLSourceReport, + manifest_lookml_constant: Optional[List[Dict[str, str]]] = [], ): self.source_config = source_config - self.looker_constant = looker_constant + self.reporter = reporter + self.manifest_lookml_constant = manifest_lookml_constant def transform(self, view: dict) -> dict: value_to_transform: Optional[str] = None @@ -258,6 +270,7 @@ def _apply_transformation(self, value: str, view: dict) -> str: return resolve_liquid_variable( text=value, liquid_variable=self.source_config.liquid_variable, + report=self.reporter, ) @@ -313,8 +326,8 @@ class LookMlIfCommentTransformer(LookMLViewTransformer): evaluate_to_true_regx: str remove_if_comment_line_regx: str - def __init__(self, source_config: LookMLSourceConfig): - super().__init__(source_config=source_config) + def __init__(self, source_config: LookMLSourceConfig, reporter: LookMLSourceReport): + super().__init__(source_config=source_config, reporter=reporter) # This regx will keep whatever after -- if looker_environment -- self.evaluate_to_true_regx = r"-- if {} --".format( @@ -355,11 +368,11 @@ def resolve_lookml_parameter(self, text: str) -> str: def replace_constants(match): key = match.group(1) - if self.looker_constant: + if self.manifest_lookml_constant: value = next( ( item["value"] - for item in self.looker_constant + for item in self.manifest_lookml_constant if item["name"] == key ), None, @@ -367,18 +380,20 @@ def replace_constants(match): if value: return value - # Resolve constant - if key in self.source_config.lookml_constant: - return str(self.source_config.lookml_constant.get(key)) + # Resolve constant from config + if key in self.source_config.lookml_constants: + return str(self.source_config.lookml_constants.get(key)) - # Check if it's a misplaced liquid variable + # Check if it's a misplaced lookml constant if key in self.source_config.liquid_variable: - logger.warning( - f"Misplaced liquid variable '@{{{key}}}' detected. Use '${{{key}}}' instead." + self.reporter.report_warning( + title="Misplaced lookml constant", + message="Misplaced lookml constant, Use 'lookml_constants' instead of 'liquid_variables'.", + context=f"Key {key}", ) return f"@{{{key}}}" - logger.warning(f"Parameter '@{{{key}}}' not found in configuration.") + logger.warning(f"Constant '@{{{key}}}' not found in configuration.") return "NULL" # Resolve @{} (constant) @@ -443,26 +458,29 @@ def view(self) -> dict: def process_lookml_template_language( source_config: LookMLSourceConfig, view_lkml_file_dict: dict, - looker_constant: Optional[List[Dict[str, str]]], + reporter: LookMLSourceReport, + manifest_lookml_constant: Optional[List[Dict[str, str]]], ) -> None: if "views" not in view_lkml_file_dict: return transformers: List[LookMLViewTransformer] = [ LookMlIfCommentTransformer( - source_config=source_config + source_config=source_config, reporter=reporter ), # First evaluate the -- if -- comments. Looker does the same LiquidVariableTransformer( - source_config=source_config + source_config=source_config, reporter=reporter ), # Now resolve liquid variables LookmlConstantTransformer( - source_config=source_config, looker_constant=looker_constant + source_config=source_config, + manifest_lookml_constant=manifest_lookml_constant, + reporter=reporter, ), # Resolve @{} constant with its corresponding value DropDerivedViewPatternTransformer( - source_config=source_config + source_config=source_config, reporter=reporter ), # Remove any ${} symbol IncompleteSqlTransformer( - source_config=source_config + source_config=source_config, reporter=reporter ), # complete any incomplete sql ] @@ -479,14 +497,16 @@ def process_lookml_template_language( def load_and_preprocess_file( path: Union[str, pathlib.Path], source_config: LookMLSourceConfig, - looker_constant: Optional[List[Dict[str, str]]] = [], + reporter: LookMLSourceReport, + manifest_lookml_constant: Optional[List[Dict[str, str]]] = [], ) -> dict: parsed = load_lkml(path) process_lookml_template_language( view_lkml_file_dict=parsed, + reporter=reporter, source_config=source_config, - looker_constant=looker_constant, + manifest_lookml_constant=manifest_lookml_constant, ) return parsed diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index fac411ac75e05..ad438fdad8859 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -165,11 +165,12 @@ class LookMLSourceConfig( "view.sql_table_name. Defaults to an empty dictionary.", ) - lookml_constant: Dict[str, str] = Field( + lookml_constants: Dict[str, str] = Field( {}, description=( - "A dictionary containing LookML constant (`@{constant_name}`) and their values. " - "Parameters are used to define dynamic references in LookML views, such as `view.sql_table_name`." + "A dictionary containing LookML constants (`@{constant_name}`) and their values. " + "If a constant is defined in the `manifest.lkml` file, its value will be used. " + "If not found in the manifest, the value from this config will be used instead. " "Defaults to an empty dictionary." ), ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 68775d28bc453..793d6019e96f4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -310,13 +310,14 @@ def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext): "manage_models permission enabled on this API key." ) from err - self.looker_constant: List[Dict[str, str]] = [] + self.manifest_lookml_constant: List[Dict[str, str]] = [] def _load_model(self, path: str) -> LookerModel: logger.debug(f"Loading model from file {path}") parsed = load_and_preprocess_file( path=path, + reporter=self.reporter, source_config=self.source_config, ) @@ -504,7 +505,9 @@ def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManife manifest_file = folder / "manifest.lkml" if manifest_file.exists(): manifest_dict = load_and_preprocess_file( - path=manifest_file, source_config=self.source_config + path=manifest_file, + source_config=self.source_config, + reporter=self.reporter, ) manifest = LookerManifest( @@ -578,7 +581,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.base_projects_folder[project] = p_ref self._recursively_check_manifests( - tmp_dir, BASE_PROJECT_NAME, visited_projects, self.looker_constant + tmp_dir, + BASE_PROJECT_NAME, + visited_projects, + self.manifest_lookml_constant, ) yield from self.get_internal_workunits() @@ -595,7 +601,7 @@ def _recursively_check_manifests( tmp_dir: str, project_name: str, project_visited: Set[str], - looker_constant: List[Dict[str, str]], + manifest_lookml_constant: List[Dict[str, str]], ) -> None: if project_name in project_visited: return @@ -613,7 +619,7 @@ def _recursively_check_manifests( return if manifest.constants: - looker_constant.extend(manifest.constants) + manifest_lookml_constant.extend(manifest.constants) # Special case handling if the root project has a name in the manifest file. if project_name == BASE_PROJECT_NAME and manifest.project_name: @@ -674,12 +680,15 @@ def _recursively_check_manifests( project_visited.add(project_name) else: self._recursively_check_manifests( - tmp_dir, remote_project.name, project_visited, looker_constant + tmp_dir, + remote_project.name, + project_visited, + manifest_lookml_constant, ) for project in manifest.local_dependencies: self._recursively_check_manifests( - tmp_dir, project, project_visited, looker_constant + tmp_dir, project, project_visited, manifest_lookml_constant ) def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 @@ -689,7 +698,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 self.base_projects_folder, self.reporter, self.source_config, - self.looker_constant, + self.manifest_lookml_constant, ) # Some views can be mentioned by multiple 'include' statements and can be included via different connections. From 0b54bb61c71d5196f7c31f9086e599cbeef651d6 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Thu, 16 Jan 2025 15:25:36 +0530 Subject: [PATCH 11/13] test: updated test for lookml constant --- .../tests/integration/lookml/test_lookml.py | 40 +- .../activity_logs.view.lkml | 9 +- .../data.model.lkml | 9 - .../data.model.lkml | 10 + .../manifest.lkml | 0 .../star_award_winner.view.lkml | 0 .../star_award_winner_dev.view.lkml | 0 .../vv_lineage_liquid_template_golden.json | 410 +------------- .../vv_lineage_lookml_constant_golden.json | 514 ++++++++++++++++++ 9 files changed, 566 insertions(+), 426 deletions(-) create mode 100644 metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/data.model.lkml rename metadata-ingestion/tests/integration/lookml/{vv-lineage-and-liquid-templates => vv-lineage-and-lookml-constant}/manifest.lkml (100%) rename metadata-ingestion/tests/integration/lookml/{vv-lineage-and-liquid-templates => vv-lineage-and-lookml-constant}/star_award_winner.view.lkml (100%) rename metadata-ingestion/tests/integration/lookml/{vv-lineage-and-liquid-templates => vv-lineage-and-lookml-constant}/star_award_winner_dev.view.lkml (100%) create mode 100644 metadata-ingestion/tests/integration/lookml/vv_lineage_lookml_constant_golden.json diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 532984f6da0d7..28155779dd852 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -18,7 +18,10 @@ load_and_preprocess_file, resolve_liquid_variable, ) -from datahub.ingestion.source.looker.lookml_config import LookMLSourceConfig +from datahub.ingestion.source.looker.lookml_config import ( + LookMLSourceConfig, + LookMLSourceReport, +) from datahub.ingestion.source.looker.lookml_refinement import LookerRefinementResolver from datahub.metadata.schema_classes import ( DatasetSnapshotClass, @@ -823,8 +826,7 @@ def test_manifest_parser(pytestconfig: pytest.Config) -> None: manifest_file = test_resources_dir / "lkml_manifest_samples/complex-manifest.lkml" manifest = load_and_preprocess_file( - path=manifest_file, - source_config=MagicMock(), + path=manifest_file, source_config=MagicMock(), reporter=LookMLSourceReport() ) assert manifest @@ -869,15 +871,11 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t "dev_database_prefix": "employee", "dev_schema_prefix": "public", }, - "_dev_database_attributes": { - "dev_database_prefix": "customer", - }, "dw_eff_dt_date": { "_is_selected": True, }, "source_region": "ap-south-1", } - new_recipe["source"]["config"]["lookml_constant"] = {"winner_table": "dev"} pipeline = Pipeline.create(new_recipe) pipeline.run() @@ -892,6 +890,31 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t ) +@freeze_time(FROZEN_TIME) +def test_view_to_view_lineage_and_lookml_constant(pytestconfig, tmp_path, mock_time): + test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" + mce_out_file = "vv_lineage_lookml_constant_golden.json" + + new_recipe = get_default_recipe( + f"{tmp_path}/{mce_out_file}", + f"{test_resources_dir}/vv-lineage-and-lookml-constant", + ) + + new_recipe["source"]["config"]["lookml_constants"] = {"winner_table": "dev"} + + pipeline = Pipeline.create(new_recipe) + pipeline.run() + pipeline.pretty_print_summary() + pipeline.raise_from_status(raise_warnings=True) + + golden_path = test_resources_dir / "vv_lineage_lookml_constant_golden.json" + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / mce_out_file, + golden_path=golden_path, + ) + + @freeze_time(FROZEN_TIME) def test_special_liquid_variables(): text: str = """{% assign source_table_variable = "source_table" | sql_quote | non_existing_filter_where_it_should_not_fail %} @@ -956,8 +979,7 @@ def test_special_liquid_variables(): # Match template after resolution of liquid variables actual_text = resolve_liquid_variable( - text=text, - liquid_variable=input_liquid_variable, + text=text, liquid_variable=input_liquid_variable, report=LookMLSourceReport() ) expected_text: str = ( diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml index 8512e1bf0469b..f0e2dec6e4678 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/activity_logs.view.lkml @@ -1,6 +1,13 @@ view: activity_logs { sql_table_name: - {{ _dev_database_attributes['dev_database_prefix'] }}_analytics.{{ _user_attributes['dev_schema_prefix'] }}_staging_app.stg_app__activity_logs;; + {% if _user_attributes['looker_env'] == 'dev' %} + {{ _user_attributes['dev_database_prefix'] }}analytics.{{ _user_attributes['dev_schema_prefix'] }}staging_app.stg_app__activity_logs + {% elsif _user_attributes['looker_env'] == 'prod' %} + analytics.staging_app.stg_app__activity_logs + {% else %} + analytics.staging_app.stg_app__activity_logs + {% endif %} + ;; dimension: generated_message_id { group_label: "IDs" diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml index 1028129a14fc5..4de4df34e15d1 100644 --- a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml @@ -10,9 +10,6 @@ include: "environment_activity_logs.view.lkml" include: "employee_income_source_as_per_env.view.lkml" include: "rent_as_employee_income_source.view.lkml" include: "child_view.view.lkml" -include: "star_award_winner.view.lkml" -include: "star_award_winner_dev.view.lkml" - explore: activity_logs { } @@ -43,9 +40,3 @@ explore: rent_as_employee_income_source { explore: child_view { } - -explore: star_award_winner { -} - -explore: star_award_winner_dev { -} diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/data.model.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/data.model.lkml new file mode 100644 index 0000000000000..6f425c469c954 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/data.model.lkml @@ -0,0 +1,10 @@ +connection: "my_connection" + +include: "star_award_winner.view.lkml" +include: "star_award_winner_dev.view.lkml" + +explore: star_award_winner { +} + +explore: star_award_winner_dev { +} diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/manifest.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/manifest.lkml similarity index 100% rename from metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/manifest.lkml rename to metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/manifest.lkml diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/star_award_winner.view.lkml similarity index 100% rename from metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml rename to metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/star_award_winner.view.lkml diff --git a/metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner_dev.view.lkml b/metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/star_award_winner_dev.view.lkml similarity index 100% rename from metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner_dev.view.lkml rename to metadata-ingestion/tests/integration/lookml/vv-lineage-and-lookml-constant/star_award_winner_dev.view.lkml diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json index 1938c029bc924..468450310c2ab 100644 --- a/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json @@ -117,7 +117,7 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "view: activity_logs {\n sql_table_name:\n {{ _dev_database_attributes['dev_database_prefix'] }}_analytics.{{ _user_attributes['dev_schema_prefix'] }}_staging_app.stg_app__activity_logs;;\n\n dimension: generated_message_id {\n group_label: \"IDs\"\n primary_key: yes\n type: number\n sql: ${TABLE}.\"GENERATED_MESSAGE_ID\" ;;\n }\n}\n", + "viewLogic": "view: activity_logs {\n sql_table_name:\n {% if _user_attributes['looker_env'] == 'dev' %}\n {{ _user_attributes['dev_database_prefix'] }}analytics.{{ _user_attributes['dev_schema_prefix'] }}staging_app.stg_app__activity_logs\n {% elsif _user_attributes['looker_env'] == 'prod' %}\n analytics.staging_app.stg_app__activity_logs\n {% else %}\n analytics.staging_app.stg_app__activity_logs\n {% endif %}\n ;;\n\n dimension: generated_message_id {\n group_label: \"IDs\"\n primary_key: yes\n type: number\n sql: ${TABLE}.\"GENERATED_MESSAGE_ID\" ;;\n }\n}\n", "viewLanguage": "lookml" } }, @@ -168,7 +168,7 @@ "time": 1586847600000, "actor": "urn:li:corpuser:datahub" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,customer_analytics.public_staging_app.stg_app__activity_logs,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,employeeanalytics.publicstaging_app.stg_app__activity_logs,PROD)", "type": "VIEW" } ], @@ -176,7 +176,7 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,customer_analytics.public_staging_app.stg_app__activity_logs,PROD),generated_message_id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,employeeanalytics.publicstaging_app.stg_app__activity_logs,PROD),generated_message_id)" ], "downstreamType": "FIELD", "downstreams": [ @@ -2827,410 +2827,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "view: star_award_winner {\n sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{customer_support_table};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n}", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.winner,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.winner,PROD),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD),id)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "star_award_winner", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "nullable": false, - "description": "", - "label": "id", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "number", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": true - } - ], - "primaryKeys": [ - "id" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "star_award_winner.view.lkml", - "looker.model": "data" - }, - "name": "star_award_winner", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", - "changeType": "UPSERT", - "aspectName": "viewProperties", - "aspect": { - "json": { - "materialized": false, - "viewLogic": "view: star_award_winner_dev {\n sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{winner_table};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n dimension: name {\n type: string\n sql: ${TABLE}.name;;\n }\n\n}", - "viewLanguage": "lookml" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", - "aspects": [ - { - "com.linkedin.pegasus2avro.common.BrowsePaths": { - "paths": [ - "/Develop/lkml_samples/" - ] - } - }, - { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false - } - }, - { - "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { - "upstreams": [ - { - "auditStamp": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD),id)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD),id)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD),name)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD),name)" - ], - "confidenceScore": 1.0 - } - ] - } - }, - { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "star_award_winner_dev", - "platform": "urn:li:dataPlatform:looker", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.OtherSchema": { - "rawSchema": "" - } - }, - "fields": [ - { - "fieldPath": "id", - "nullable": false, - "description": "", - "label": "id", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "number", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": true - }, - { - "fieldPath": "name", - "nullable": false, - "description": "", - "label": "", - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "string", - "recursive": false, - "globalTags": { - "tags": [ - { - "tag": "urn:li:tag:Dimension" - } - ] - }, - "isPartOfKey": false - } - ], - "primaryKeys": [ - "id" - ] - } - }, - { - "com.linkedin.pegasus2avro.dataset.DatasetProperties": { - "customProperties": { - "looker.file.path": "star_award_winner_dev.view.lkml", - "looker.model": "data" - }, - "name": "star_award_winner_dev", - "tags": [] - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Develop" - }, - { - "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", - "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "lookml-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "tag", "entityUrn": "urn:li:tag:Dimension", diff --git a/metadata-ingestion/tests/integration/lookml/vv_lineage_lookml_constant_golden.json b/metadata-ingestion/tests/integration/lookml/vv_lineage_lookml_constant_golden.json new file mode 100644 index 0000000000000..e8e65f49df623 --- /dev/null +++ b/metadata-ingestion/tests/integration/lookml/vv_lineage_lookml_constant_golden.json @@ -0,0 +1,514 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "looker", + "env": "PROD", + "project_name": "lkml_samples" + }, + "name": "lkml_samples", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:looker" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "LookML Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Folders" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: star_award_winner {\n sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{customer_support_table};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n}", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.winner,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.winner,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD),id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "star_award_winner", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "id", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": true + } + ], + "primaryKeys": [ + "id" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "star_award_winner.view.lkml", + "looker.model": "data" + }, + "name": "star_award_winner", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "view: star_award_winner_dev {\n sql_table_name: @{customer_support_db}.@{customer_support_schema}.@{winner_table};;\n\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n dimension: name {\n type: string\n sql: ${TABLE}.name;;\n }\n\n}", + "viewLanguage": "lookml" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/Develop/lkml_samples/" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1586847600000, + "actor": "urn:li:corpuser:datahub" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,star_award_winner_year.public.dev,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD),name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "star_award_winner_dev", + "platform": "urn:li:dataPlatform:looker", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": false, + "description": "", + "label": "id", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "number", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": true + }, + { + "fieldPath": "name", + "nullable": false, + "description": "", + "label": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Dimension" + } + ] + }, + "isPartOfKey": false + } + ], + "primaryKeys": [ + "id" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "looker.file.path": "star_award_winner_dev.view.lkml", + "looker.model": "data" + }, + "name": "star_award_winner_dev", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner_dev,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Develop" + }, + { + "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e", + "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Dimension", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Dimension" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "lookml-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file From f08b82798faf6c12098d206c7d26e4463c593f26 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Fri, 17 Jan 2025 13:30:12 +0530 Subject: [PATCH 12/13] fix: pr comment --- .../docs/sources/looker/lookml_post.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/docs/sources/looker/lookml_post.md b/metadata-ingestion/docs/sources/looker/lookml_post.md index 77893ff2f17b8..d3b963a060461 100644 --- a/metadata-ingestion/docs/sources/looker/lookml_post.md +++ b/metadata-ingestion/docs/sources/looker/lookml_post.md @@ -8,20 +8,20 @@ db: ANALYTICS_PROD ``` -2. If a view contains a LookML constant (e.g., `sql_table_name: @{db}.kafka_streaming.events;`), its value is resolved in the following order: +2. If your LookML code references a constant (e.g., `sql_table_name: @{db}.kafka_streaming.events;`), its value is resolved in the following order: - **First, checks the `manifest.lkml` file** for the constant definition. - ```manifest.lkml - constant: db { - value: "ANALYTICS_PROD" - } - ``` + ```manifest.lkml + constant: db { + value: "ANALYTICS_PROD" + } + ``` - **If not found, falls back to `config`** ```yml lookml_constants: db: ANALYTICS_PROD - ``` + ``` ### Multi-Project LookML (Advanced) From f1daf12f21ba547d03b80d7ddd2cc279e2ce984d Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Sat, 18 Jan 2025 20:26:26 +0530 Subject: [PATCH 13/13] fix: rename liquid_variable -> liquid_variables --- metadata-ingestion/docs/sources/looker/looker_recipe.yml | 2 +- metadata-ingestion/docs/sources/looker/lookml_post.md | 4 ++-- .../ingestion/source/looker/looker_template_language.py | 4 ++-- .../src/datahub/ingestion/source/looker/lookml_config.py | 6 +++++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/docs/sources/looker/looker_recipe.yml b/metadata-ingestion/docs/sources/looker/looker_recipe.yml index 388824df20052..7b4b53448fa23 100644 --- a/metadata-ingestion/docs/sources/looker/looker_recipe.yml +++ b/metadata-ingestion/docs/sources/looker/looker_recipe.yml @@ -9,7 +9,7 @@ source: client_secret: ${LOOKER_CLIENT_SECRET} # Liquid variables - # liquid_variable: + # liquid_variables: # _user_attributes: # looker_env: "dev" # dev_database_prefix: "employee" diff --git a/metadata-ingestion/docs/sources/looker/lookml_post.md b/metadata-ingestion/docs/sources/looker/lookml_post.md index d3b963a060461..ee7070de220ab 100644 --- a/metadata-ingestion/docs/sources/looker/lookml_post.md +++ b/metadata-ingestion/docs/sources/looker/lookml_post.md @@ -1,9 +1,9 @@ #### Configuration Notes -1. If a view contains a liquid template (e.g. `sql_table_name: {{ user_attributes['db']}}.kafka_streaming.events }}`, with `db=ANALYTICS_PROD`), then you will need to specify the values of those variables in the `liquid_variable` config as shown below: +1. If a view contains a liquid template (e.g. `sql_table_name: {{ user_attributes['db']}}.kafka_streaming.events }}`, with `db=ANALYTICS_PROD`), then you will need to specify the values of those variables in the `liquid_variables` config as shown below: ```yml - liquid_variable: + liquid_variables: user_attributes: db: ANALYTICS_PROD ``` diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index 99ed2bd0aef18..478a6ca687b29 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -269,7 +269,7 @@ class LiquidVariableTransformer(LookMLViewTransformer): def _apply_transformation(self, value: str, view: dict) -> str: return resolve_liquid_variable( text=value, - liquid_variable=self.source_config.liquid_variable, + liquid_variable=self.source_config.liquid_variables, report=self.reporter, ) @@ -385,7 +385,7 @@ def replace_constants(match): return str(self.source_config.lookml_constants.get(key)) # Check if it's a misplaced lookml constant - if key in self.source_config.liquid_variable: + if key in self.source_config.liquid_variables: self.reporter.report_warning( title="Misplaced lookml constant", message="Misplaced lookml constant, Use 'lookml_constants' instead of 'liquid_variables'.", diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index ad438fdad8859..543f280910eb1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -158,13 +158,17 @@ class LookMLSourceConfig( description="When enabled, looker refinement will be processed to adapt an existing view.", ) - liquid_variable: Dict[Any, Any] = Field( + liquid_variables: Dict[Any, Any] = Field( {}, description="A dictionary containing Liquid variables, Liquid logic, and LookML parameters with their corresponding values, utilized in SQL-defined " "derived views. The Liquid template will be resolved in view.derived_table.sql and " "view.sql_table_name. Defaults to an empty dictionary.", ) + _liquid_variable_deprecated = pydantic_renamed_field( + old_name="liquid_variable", new_name="liquid_variables", print_warning=True + ) + lookml_constants: Dict[str, str] = Field( {}, description=(