feat(ingestion/lookml): resolve access notation for LookML Constant

metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py

@@ -198,9 +198,9 @@
     def transform(self, view: dict) -> dict:
         value_to_transform: Optional[str] = None
 
-        # is_attribute_supported check is required because not all transformer works on all attributes in current
-        # case mostly all transformer works on sql_table_name and derived.sql attributes,
-        # however IncompleteSqlTransformer only transform the derived.sql attribute
+        # is_attribute_supported check is required because not all transformers work on all attributes in the current
+        #  case, mostly all transformers work on sql_table_name and derived.sql attributes;
+        # however, IncompleteSqlTransformer only transform the derived.sql attribute
         if SQL_TABLE_NAME in view and self.is_attribute_supported(SQL_TABLE_NAME):
             # Give precedence to already processed transformed view.sql_table_name to apply more transformation
             value_to_transform = view.get(
@@ -287,7 +287,7 @@
 
 class DropDerivedViewPatternTransformer(LookMLViewTransformer):
     """
-    drop ${} from datahub_transformed_sql_table_name and  view["derived_table"]["datahub_transformed_sql_table_name"] values.
+    drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values.
 
     Example: transform ${employee_income_source.SQL_TABLE_NAME} to employee_income_source.SQL_TABLE_NAME
     """
@@ -335,6 +335,56 @@
         return self._apply_regx(value)
 
 
+class LookmlParameterTransformer(LookMLViewTransformer):
+    """
+    Replace LookML parameters (@{param} or ${var}) from the configuration.
+    """
+
+    PARAMETER_PATTERN = r"@{(\w+)}"  # Matches @{param}
+    LIQUID_VARIABLE_PATTERN = r"\${(\w+)}"  # Matches ${var}
+
+    def resolve_lookml_parameter(self, text: str) -> str:
+        """
+        Resolves LookML parameters (@{param}) and liquid variables (${var}).
+        Logs warnings for misplaced or missing variables.
+        """
+
+        def replace_parameters(match):
+            key = match.group(1)
+            # Check if it's a misplaced liquid variable
+            if key in self.source_config.liquid_variable:
+                logger.warning(
+                    f"Misplaced liquid variable '@{{{key}}}' detected. Use '${{{key}}}' instead."
+                )
+                return f"@{{{key}}}"
+
+            # Resolve parameter
+            if key in self.source_config.lookml_parameter:
+                return str(self.source_config.lookml_parameter.get(key))
+
+            logger.warning(f"Parameter '@{{{key}}}' not found in configuration.")
+            return ""
+
+        def replace_liquid_variables(match):
+            key = match.group(1)
+            # Resolve liquid variable
+            if key in self.source_config.liquid_variable:
+                return str(self.source_config.liquid_variable.get(key))
+
+            logger.warning(f"Liquid variable '${{{key}}}' not found in configuration.")
+            return ""
+
+        # Resolve @{param} (parameters)
+        text = re.sub(self.PARAMETER_PATTERN, replace_parameters, text)
+
+        # Resolve ${var} (liquid variables)
+        text = re.sub(self.LIQUID_VARIABLE_PATTERN, replace_liquid_variables, text)
+        return text
+
+    def _apply_transformation(self, value: str, view: dict) -> str:
+        return self.resolve_lookml_parameter(text=value)
+
+
 class TransformedLookMlView:
     """
     TransformedLookMlView is collecting output of LookMLViewTransformer and creating a new transformed LookML view.
@@ -401,6 +451,9 @@
         LiquidVariableTransformer(
             source_config=source_config
         ),  # Now resolve liquid variables
+        LookmlParameterTransformer(
+            source_config=source_config
+        ),  # Remove @{param}/${var} with its corresponding value
         DropDerivedViewPatternTransformer(
             source_config=source_config
         ),  # Remove any ${} symbol

metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py

@@ -165,6 +165,15 @@ class LookMLSourceConfig(
         "view.sql_table_name. Defaults to an empty dictionary.",
     )
 
+    lookml_parameter: Dict[Any, Any] = Field(
+        {},
+        description=(
+            "A dictionary containing LookML parameters (`@{param_name}`) and their values. "
+            "Parameters are used to define dynamic references in LookML views, such as `view.sql_table_name`."
+            "Defaults to an empty dictionary."
+        ),
+    )
+
     looker_environment: Literal["prod", "dev"] = Field(
         "prod",
         description="A looker prod or dev environment. "

metadata-ingestion/tests/integration/lookml/test_lookml.py

@@ -873,6 +873,10 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t
             "_is_selected": True,
         },
         "source_region": "ap-south-1",
+        "db": "test-db",
+    }
+    new_recipe["source"]["config"]["lookml_parameter"] = {
+        "star_award_winner_year": "public.winner_2025"
     }
 
     pipeline = Pipeline.create(new_recipe)

metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml

@@ -10,6 +10,7 @@ include: "environment_activity_logs.view.lkml"
 include: "employee_income_source_as_per_env.view.lkml"
 include: "rent_as_employee_income_source.view.lkml"
 include: "child_view.view.lkml"
+include: "star_award_winner.view.lkml"
 
 explore: activity_logs {
 }
@@ -39,4 +40,7 @@ explore: rent_as_employee_income_source {
 }
 
 explore: child_view {
+}
+
+explore: star_award_winner {
 }

metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml

@@ -0,0 +1,24 @@
+view: star_award_winner {
+  sql_table_name: ${db}.@{star_award_winner_year};;
+
+
+  dimension: id {
+    label: "id"
+    primary_key: yes
+    type: number
+    sql: ${TABLE}.id ;;
+  }
+
+  parameter: star_award_winner_year {
+    type: string
+    allowed_value: {
+      label: "Star Award Winner Of 2025"
+      value: "public.winner_2025"
+    }
+    allowed_value: {
+      label: "Star Award Winner Of 2024"
+      value: "public.winner_2024"
+    }
+  }
+
+}

metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json

@@ -2827,6 +2827,192 @@
         "lastRunId": "no-run-id-provided"
     }
 },
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "subTypes",
+    "aspect": {
+        "json": {
+            "typeNames": [
+                "View"
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "viewProperties",
+    "aspect": {
+        "json": {
+            "materialized": false,
+            "viewLogic": "view: star_award_winner {\n  sql_table_name: ${db}.@{star_award_winner_year};;\n\n\n  dimension: id {\n    label: \"id\"\n    primary_key: yes\n    type: number\n    sql: ${TABLE}.id ;;\n  }\n\n  parameter: star_award_winner_year {\n    type: string\n    allowed_value: {\n      label: \"Star Award Winner Of 2025\"\n      value: \"public.winner_2025\"\n    }\n    allowed_value: {\n      label: \"Star Award Winner Of 2024\"\n      value: \"public.winner_2024\"\n    }\n  }\n\n}",
+            "viewLanguage": "lookml"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "container",
+    "aspect": {
+        "json": {
+            "container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "proposedSnapshot": {
+        "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
+            "urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
+            "aspects": [
+                {
+                    "com.linkedin.pegasus2avro.common.BrowsePaths": {
+                        "paths": [
+                            "/Develop/lkml_samples/"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.common.Status": {
+                        "removed": false
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
+                        "upstreams": [
+                            {
+                                "auditStamp": {
+                                    "time": 1586847600000,
+                                    "actor": "urn:li:corpuser:datahub"
+                                },
+                                "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,test-db.public.winner_2025,PROD)",
+                                "type": "VIEW"
+                            }
+                        ],
+                        "fineGrainedLineages": [
+                            {
+                                "upstreamType": "FIELD_SET",
+                                "upstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,test-db.public.winner_2025,PROD),id)"
+                                ],
+                                "downstreamType": "FIELD",
+                                "downstreams": [
+                                    "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD),id)"
+                                ],
+                                "confidenceScore": 1.0
+                            }
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
+                        "schemaName": "star_award_winner",
+                        "platform": "urn:li:dataPlatform:looker",
+                        "version": 0,
+                        "created": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "lastModified": {
+                            "time": 0,
+                            "actor": "urn:li:corpuser:unknown"
+                        },
+                        "hash": "",
+                        "platformSchema": {
+                            "com.linkedin.pegasus2avro.schema.OtherSchema": {
+                                "rawSchema": ""
+                            }
+                        },
+                        "fields": [
+                            {
+                                "fieldPath": "id",
+                                "nullable": false,
+                                "description": "",
+                                "label": "id",
+                                "type": {
+                                    "type": {
+                                        "com.linkedin.pegasus2avro.schema.NumberType": {}
+                                    }
+                                },
+                                "nativeDataType": "number",
+                                "recursive": false,
+                                "globalTags": {
+                                    "tags": [
+                                        {
+                                            "tag": "urn:li:tag:Dimension"
+                                        }
+                                    ]
+                                },
+                                "isPartOfKey": true
+                            }
+                        ],
+                        "primaryKeys": [
+                            "id"
+                        ]
+                    }
+                },
+                {
+                    "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
+                        "customProperties": {
+                            "looker.file.path": "star_award_winner.view.lkml",
+                            "looker.model": "data"
+                        },
+                        "name": "star_award_winner",
+                        "tags": []
+                    }
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "browsePathsV2",
+    "aspect": {
+        "json": {
+            "path": [
+                {
+                    "id": "Develop"
+                },
+                {
+                    "id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
+                    "urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
+                }
+            ]
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1586847600000,
+        "runId": "lookml-test",
+        "lastRunId": "no-run-id-provided"
+    }
+},
 {
     "entityType": "tag",
     "entityUrn": "urn:li:tag:Dimension",