Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ingestion/lookml): resolve access notation for LookML parameters #12172

Closed
wants to merge 6 commits into from
Closed

feat(ingestion/lookml): resolve access notation for LookML parameters #12172

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0cd9551
lookml parameter support
sid-acryl Dec 19, 2024
6422b27
Merge branch 'master' into ing-784-lookml-parameter
sid-acryl Dec 19, 2024
092de33
Merge branch 'master' into ing-784-lookml-parameter
sid-acryl Dec 19, 2024
a0fc80b
Merge branch 'master' into ing-784-lookml-parameter
sid-acryl Dec 20, 2024
f1723d4
Merge branch 'master' into ing-784-lookml-parameter
sid-acryl Dec 20, 2024
6fef91d
Merge branch 'master' into ing-784-lookml-parameter
anshbansal Dec 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 27 additions & 4 deletions metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,9 +198,9 @@ def __init__(self, source_config: LookMLSourceConfig):
def transform(self, view: dict) -> dict:
value_to_transform: Optional[str] = None

# is_attribute_supported check is required because not all transformer works on all attributes in current
# case mostly all transformer works on sql_table_name and derived.sql attributes,
# however IncompleteSqlTransformer only transform the derived.sql attribute
# is_attribute_supported check is required because not all transformers work on all attributes in the current
# case, mostly all transformers work on sql_table_name and derived.sql attributes;
# however, IncompleteSqlTransformer only transform the derived.sql attribute
if SQL_TABLE_NAME in view and self.is_attribute_supported(SQL_TABLE_NAME):
# Give precedence to already processed transformed view.sql_table_name to apply more transformation
value_to_transform = view.get(
Expand Down Expand Up @@ -287,7 +287,7 @@ def _apply_transformation(self, value: str, view: dict) -> str:

class DropDerivedViewPatternTransformer(LookMLViewTransformer):
"""
drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values.
drop ${} from datahub_transformed_sql_table_name and view["derived_table"]["datahub_transformed_sql_table_name"] values.

Example: transform ${employee_income_source.SQL_TABLE_NAME} to employee_income_source.SQL_TABLE_NAME
"""
Expand Down Expand Up @@ -335,6 +335,26 @@ def _apply_transformation(self, value: str, view: dict) -> str:
return self._apply_regx(value)


class LookmlParameterTransformer(LookMLViewTransformer):
"""
Replace the lookml parameter @{variable} with their values.
"""

PATTERN = r"@{(\w+)}"

def resolve_lookml_parameter(self, text: str) -> str:
return re.sub(
LookmlParameterTransformer.PATTERN,
lambda match: self.source_config.liquid_variable.get(
match.group(1), match.group(0)
), # Replace or keep original
text,
)

def _apply_transformation(self, value: str, view: dict) -> str:
return self.resolve_lookml_parameter(text=value)


class TransformedLookMlView:
"""
TransformedLookMlView is collecting output of LookMLViewTransformer and creating a new transformed LookML view.
Expand Down Expand Up @@ -401,6 +421,9 @@ def process_lookml_template_language(
LiquidVariableTransformer(
source_config=source_config
), # Now resolve liquid variables
LookmlParameterTransformer(
source_config=source_config
), # Remove @{variable} with its corresponding value
DropDerivedViewPatternTransformer(
source_config=source_config
), # Remove any ${} symbol
Expand Down
1 change: 1 addition & 0 deletions metadata-ingestion/tests/integration/lookml/test_lookml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -873,6 +873,7 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t
"_is_selected": True,
},
"source_region": "ap-south-1",
"star_award_winner_year": "public.winner_2025",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this also a liquid variable? or is it something else?

We should also add some docs around the liquid variable resolution functionality

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The parameter field, as described in the Looker documentation, is used to define variables accessible within Liquid templates using {% ... %} or {{ ... }} syntax. If no complex conditions need to be evaluated, these parameters can also be directly referenced in LookML views using the @{ ... } notation.

For example:

  • sql_table_name: @{star_award_winner_year}
  • sql_table_name: {% if user_attribute('use_schema') == 'true' %} {{ star_award_winner_year }} {% else %} public.winner_2024 {% endif %} ;;

The key difference between @{ ... } and using a Liquid template is that Liquid allows access to additional user-related attributes, while @{ ... } only references parameters defined in the parameter attribute of the view.

The attribute name liquid_variable in recipe is bit confusing. We can rename it to lookml_parameter, then it wouldn't convey the dynamic nature of liquid template. I can add additional documentation to liquid_variable or you can suggest some other name

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is actually incorrect - the @{ variable } syntax is for looker constants, which are defined in the manifest.lkml https://cloud.google.com/looker/docs/reference/param-manifest-constant?hl=en

The implementation here conflates parameters and constants in a way that is likely extremely confusing to the end user

}

pipeline = Pipeline.create(new_recipe)
Expand Down
4 changes: 4 additions & 0 deletions metadata-ingestion/tests/integration/lookml/vv-lineage-and-liquid-templates/data.model.lkml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ include: "environment_activity_logs.view.lkml"
include: "employee_income_source_as_per_env.view.lkml"
include: "rent_as_employee_income_source.view.lkml"
include: "child_view.view.lkml"
include: "star_award_winner.view.lkml"

explore: activity_logs {
}
Expand Down Expand Up @@ -39,4 +40,7 @@ explore: rent_as_employee_income_source {
}

explore: child_view {
}

explore: star_award_winner {
}
23 changes: 23 additions & 0 deletions ...tion/tests/integration/lookml/vv-lineage-and-liquid-templates/star_award_winner.view.lkml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
view: star_award_winner {
sql_table_name: @{star_award_winner_year} ;;

dimension: id {
label: "id"
primary_key: yes
type: number
sql: ${TABLE}.id ;;
}

parameter: star_award_winner_year {
type: string
allowed_value: {
label: "Star Award Winner Of 2025"
value: "public.winner_2025"
}
allowed_value: {
label: "Star Award Winner Of 2024"
value: "public.winner_2024"
}
}

}
186 changes: 186 additions & 0 deletions metadata-ingestion/tests/integration/lookml/vv_lineage_liquid_template_golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2827,6 +2827,192 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"View"
]
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
"changeType": "UPSERT",
"aspectName": "viewProperties",
"aspect": {
"json": {
"materialized": false,
"viewLogic": "view: star_award_winner {\n sql_table_name: @{star_award_winner_year} ;;\n\n dimension: id {\n label: \"id\"\n primary_key: yes\n type: number\n sql: ${TABLE}.id ;;\n }\n\n parameter: star_award_winner_year {\n type: string\n allowed_value: {\n label: \"Star Award Winner Of 2025\"\n value: \"public.winner_2025\"\n }\n allowed_value: {\n label: \"Star Award Winner Of 2024\"\n value: \"public.winner_2024\"\n }\n }\n\n}",
"viewLanguage": "lookml"
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.common.BrowsePaths": {
"paths": [
"/Develop/lkml_samples/"
]
}
},
{
"com.linkedin.pegasus2avro.common.Status": {
"removed": false
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": [
{
"auditStamp": {
"time": 1586847600000,
"actor": "urn:li:corpuser:datahub"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD)",
"type": "VIEW"
}
],
"fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,.public.winner_2025,PROD),id)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD),id)"
],
"confidenceScore": 1.0
}
]
}
},
{
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "star_award_winner",
"platform": "urn:li:dataPlatform:looker",
"version": 0,
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"hash": "",
"platformSchema": {
"com.linkedin.pegasus2avro.schema.OtherSchema": {
"rawSchema": ""
}
},
"fields": [
{
"fieldPath": "id",
"nullable": false,
"description": "",
"label": "id",
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
}
},
"nativeDataType": "number",
"recursive": false,
"globalTags": {
"tags": [
{
"tag": "urn:li:tag:Dimension"
}
]
},
"isPartOfKey": true
}
],
"primaryKeys": [
"id"
]
}
},
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"looker.file.path": "star_award_winner.view.lkml",
"looker.model": "data"
},
"name": "star_award_winner",
"tags": []
}
}
]
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,lkml_samples.view.star_award_winner,PROD)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "Develop"
},
{
"id": "urn:li:container:78f22c19304954b15e8adb1d9809975e",
"urn": "urn:li:container:78f22c19304954b15e8adb1d9809975e"
}
]
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:Dimension",
Expand Down
Loading