From 5ff6295b0fa7fcc04f3006da0d5a8c7aeb995402 Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Wed, 13 Nov 2024 23:48:30 -0800 Subject: [PATCH 001/174] fix(ingest/partition-executor): Fix deadlock by recomputing ready items (#11853) --- metadata-ingestion/setup.py | 1 + .../datahub/utilities/partition_executor.py | 25 ++++++++++++------- .../unit/utilities/test_partition_executor.py | 17 ++++++++++--- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index b1e1d7044a3396..2014d8ca4e4ddc 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -573,6 +573,7 @@ test_api_requirements = { "pytest>=6.2.2", + "pytest-timeout", # Missing numpy requirement in 8.0.0 "deepdiff!=8.0.0", "PyYAML", diff --git a/metadata-ingestion/src/datahub/utilities/partition_executor.py b/metadata-ingestion/src/datahub/utilities/partition_executor.py index aeabc5c55e6b2d..0f0e9784464f6d 100644 --- a/metadata-ingestion/src/datahub/utilities/partition_executor.py +++ b/metadata-ingestion/src/datahub/utilities/partition_executor.py @@ -237,6 +237,11 @@ def __init__( process_batch: Callable[[List], None], max_per_batch: int = 100, min_process_interval: timedelta = _DEFAULT_BATCHER_MIN_PROCESS_INTERVAL, + # Why 3 seconds? It's somewhat arbitrary. + # We don't want it to be too high, since then liveness suffers, + # particularly during a dirty shutdown. If it's too low, then we'll + # waste CPU cycles rechecking the timer, only to call get again. + read_from_pending_interval: timedelta = timedelta(seconds=3), ) -> None: """Similar to PartitionExecutor, but with batching. @@ -262,8 +267,10 @@ def __init__( self.max_per_batch = max_per_batch self.process_batch = process_batch self.min_process_interval = min_process_interval + self.read_from_pending_interval = read_from_pending_interval assert self.max_workers > 1 + self.state_lock = threading.Lock() self._executor = ThreadPoolExecutor( # We add one here to account for the clearinghouse worker thread. max_workers=max_workers + 1, @@ -362,12 +369,8 @@ def _build_batch() -> List[_BatchPartitionWorkItem]: if not blocking: next_item = self._pending.get_nowait() else: - # Why 3 seconds? It's somewhat arbitrary. - # We don't want it to be too high, since then liveness suffers, - # particularly during a dirty shutdown. If it's too low, then we'll - # waste CPU cycles rechecking the timer, only to call get again. next_item = self._pending.get( - timeout=3, # seconds + timeout=self.read_from_pending_interval.total_seconds(), ) if next_item is None: # None is the shutdown signal @@ -379,6 +382,9 @@ def _build_batch() -> List[_BatchPartitionWorkItem]: pending_key_completion.append(next_item) else: next_batch.append(next_item) + + if not next_batch: + next_batch = _find_ready_items() except queue.Empty: if not blocking: break @@ -452,10 +458,11 @@ def _ensure_clearinghouse_started(self) -> None: f"{self.__class__.__name__} is shutting down; cannot submit new work items." ) - # Lazily start the clearinghouse worker. - if not self._clearinghouse_started: - self._clearinghouse_started = True - self._executor.submit(self._clearinghouse_worker) + with self.state_lock: + # Lazily start the clearinghouse worker. + if not self._clearinghouse_started: + self._clearinghouse_started = True + self._executor.submit(self._clearinghouse_worker) def submit( self, diff --git a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py index 2901b4bba107ea..e3a68405e3c0ac 100644 --- a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py +++ b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py @@ -1,7 +1,11 @@ import logging +import math import time from concurrent.futures import Future +import pytest +from pydantic.schema import timedelta + from datahub.utilities.partition_executor import ( BatchPartitionExecutor, PartitionExecutor, @@ -129,7 +133,9 @@ def process_batch(batch): } +@pytest.mark.timeout(10) def test_batch_partition_executor_max_batch_size(): + n = 20 # Exceed max_pending to test for deadlocks when max_pending exceeded batches_processed = [] def process_batch(batch): @@ -137,15 +143,20 @@ def process_batch(batch): time.sleep(0.1) # Simulate batch processing time with BatchPartitionExecutor( - max_workers=5, max_pending=20, process_batch=process_batch, max_per_batch=2 + max_workers=5, + max_pending=10, + process_batch=process_batch, + max_per_batch=2, + min_process_interval=timedelta(seconds=1), + read_from_pending_interval=timedelta(seconds=1), ) as executor: # Submit more tasks than the max_per_batch to test batching limits. - for i in range(5): + for i in range(n): executor.submit("key3", "key3", f"task{i}") # Check the batches. logger.info(f"batches_processed: {batches_processed}") - assert len(batches_processed) == 3 + assert len(batches_processed) == math.ceil(n / 2), "Incorrect number of batches" for batch in batches_processed: assert len(batch) <= 2, "Batch size exceeded max_per_batch limit" From ab0b0a2eb907f327e267ce0b61e33eff2ea06470 Mon Sep 17 00:00:00 2001 From: Jonny Dixon <45681293+acrylJonny@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:17:50 +0000 Subject: [PATCH 002/174] fix(ingest/dremio): Dremio software jobs retrieval SQL query fix query error (#11817) Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../ingestion/source/dremio/dremio_api.py | 2 +- .../source/dremio/dremio_sql_queries.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py index e28eb08e492ee2..db83dde7cf6131 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py @@ -249,7 +249,7 @@ def post(self, url: str, data: str) -> Dict: ) return response.json() - def execute_query(self, query: str, timeout: int = 300) -> List[Dict[str, Any]]: + def execute_query(self, query: str, timeout: int = 3600) -> List[Dict[str, Any]]: """Execute SQL query with timeout and error handling""" try: response = self.post(url="/sql", data=json.dumps({"sql": query})) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py index 1c247c7d1f7bc6..161e8141c8852d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py @@ -235,21 +235,34 @@ class DremioSQLQueries: TABLE_NAME ASC """ + # Dremio Documentation: https://docs.dremio.com/current/reference/sql/system-tables/jobs_recent/ + # queried_datasets incorrectly documented as [varchar]. Observed as varchar. + # LENGTH used as opposed to ARRAY_SIZE QUERY_ALL_JOBS = """ SELECT - * + job_id, + user_name, + submitted_ts, + query, + queried_datasets FROM SYS.JOBS_RECENT WHERE STATUS = 'COMPLETED' - AND ARRAY_SIZE(queried_datasets)>0 + AND LENGTH(queried_datasets)>0 AND user_name != '$dremio$' AND query_type not like '%INTERNAL%' """ + # Dremio Documentation: https://docs.dremio.com/cloud/reference/sql/system-tables/jobs-historical + # queried_datasets correctly documented as [varchar] QUERY_ALL_JOBS_CLOUD = """ SELECT - * + job_id, + user_name, + submitted_ts, + query, + CONCAT('[', ARRAY_TO_STRING(queried_datasets, ','), ']') as queried_datasets FROM sys.project.history.jobs WHERE From e32459ff6b239653b3e0708c96bcdd0c725599e1 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 14 Nov 2024 10:06:39 -0600 Subject: [PATCH 003/174] fix(docs): Update v_0_3_7.md (#11861) --- docs/managed-datahub/release-notes/v_0_3_7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index dc1c702c89fb23..dab9d0d0ae9dac 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -19,7 +19,7 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies - Breaking Changes - Authentication & RestAPI Authorization enabled by default (since v0.3.6) - - Helm Chart Requirement: 1.4.136+ + - Helm Chart Requirement: 1.4.137+ - Recommend setting timezone for `datahub-gc` and `datahub-usage-reporting` - ```yaml acryl-datahub: From 2301bd92728d015506a7eda6a54c44325bde2b00 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Fri, 15 Nov 2024 17:19:25 +0530 Subject: [PATCH 004/174] fix(doc): fix link to doc, update cli recommendation (#11866) --- docs/managed-datahub/release-notes/v_0_3_7.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index dab9d0d0ae9dac..59b7a23b5e5836 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -7,7 +7,7 @@ Release Availability Date Recommended CLI/SDK --- -- `v0.14.1.7` with release notes at https://github.com/datahub-project/datahub/releases/tag/v0.14.1.7 +- `v0.14.1.11` with release notes at https://github.com/datahub/datahub/releases/tag/v0.14.1.11 If you are using an older CLI/SDK version, then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, GitHub Actions, Airflow, in Python SDK somewhere, Java SDK, etc. This is a strong recommendation to upgrade, as we keep on pushing fixes in the CLI, and it helps us support you better. @@ -91,7 +91,7 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies - Improved UX for setting up and managing SSO - Ingestion changes - - In addition to the improvements listed here: https://github.com/datahub-project/datahub/releases/tag/v0.14.1.7 + - In addition to the improvements listed here: https://github.com/acryldata/datahub/releases/tag/v0.14.1.11 - PowerBI: Support for PowerBI Apps and cross-workspace lineage - Fivetran: Major improvements to configurability and improved reliability with large Fivetran setups - Snowflake & BigQuery: Improved handling of temporary tables and swap statements when generating lineage From abce77e101953c0707cf8e825f58e5ce42643af3 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 15 Nov 2024 06:34:04 -0600 Subject: [PATCH 005/174] chore(version): bump netty version (#11862) --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 77f8395ac898e0..6e6dadb7ebfa34 100644 --- a/build.gradle +++ b/build.gradle @@ -393,7 +393,7 @@ subprojects { implementation externalDependency.annotationApi constraints { implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion") - implementation('io.netty:netty-all:4.1.114.Final') + implementation('io.netty:netty-all:4.1.115.Final') implementation('org.apache.commons:commons-compress:1.27.1') implementation('org.apache.velocity:velocity-engine-core:2.4') implementation('org.hibernate:hibernate-validator:6.0.20.Final') From becda8fd34bdce97ff1c6dd41dce7074a1949cbd Mon Sep 17 00:00:00 2001 From: skrydal Date: Fri, 15 Nov 2024 15:49:45 +0100 Subject: [PATCH 006/174] docs(ingest): transformer docs and java example (#11859) --- metadata-ingestion/docs/transformer/intro.md | 7 +++++++ metadata-integration/java/as-a-library.md | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/docs/transformer/intro.md b/metadata-ingestion/docs/transformer/intro.md index 0a766f7fd747d4..85ba42fca46745 100644 --- a/metadata-ingestion/docs/transformer/intro.md +++ b/metadata-ingestion/docs/transformer/intro.md @@ -26,6 +26,8 @@ DataHub provided transformers for dataset are: - [Simple Add Dataset ownership](./dataset_transformer.md#simple-add-dataset-ownership) - [Pattern Add Dataset ownership](./dataset_transformer.md#pattern-add-dataset-ownership) - [Simple Remove Dataset ownership](./dataset_transformer.md#simple-remove-dataset-ownership) +- [Extract Ownership from Tags](./dataset_transformer.md#extract-ownership-from-tags) +- [Clean suffix prefix from Ownership](./dataset_transformer.md#clean-suffix-prefix-from-ownership) - [Mark Dataset Status](./dataset_transformer.md#mark-dataset-status) - [Simple Add Dataset globalTags](./dataset_transformer.md#simple-add-dataset-globaltags) - [Pattern Add Dataset globalTags](./dataset_transformer.md#pattern-add-dataset-globaltags) @@ -33,9 +35,14 @@ DataHub provided transformers for dataset are: - [Set Dataset browsePath](./dataset_transformer.md#set-dataset-browsepath) - [Simple Add Dataset glossaryTerms](./dataset_transformer.md#simple-add-dataset-glossaryterms) - [Pattern Add Dataset glossaryTerms](./dataset_transformer.md#pattern-add-dataset-glossaryterms) +- [Add Dataset globalTags](./dataset_transformer.md#add-dataset-globaltags) - [Pattern Add Dataset Schema Field glossaryTerms](./dataset_transformer.md#pattern-add-dataset-schema-field-glossaryterms) - [Pattern Add Dataset Schema Field globalTags](./dataset_transformer.md#pattern-add-dataset-schema-field-globaltags) - [Simple Add Dataset datasetProperties](./dataset_transformer.md#simple-add-dataset-datasetproperties) - [Add Dataset datasetProperties](./dataset_transformer.md#add-dataset-datasetproperties) - [Simple Add Dataset domains](./dataset_transformer.md#simple-add-dataset-domains) - [Pattern Add Dataset domains](./dataset_transformer.md#pattern-add-dataset-domains) +- [Domain Mapping Based on Tags](./dataset_transformer.md#domain-mapping-based-on-tags) +- [Simple Add Dataset dataProduct ](./dataset_transformer.md#simple-add-dataset-dataproduct) +- [Pattern Add Dataset dataProduct](./dataset_transformer.md#pattern-add-dataset-dataproduct) +- [Add Dataset dataProduct](./dataset_transformer.md#add-dataset-dataproduct) diff --git a/metadata-integration/java/as-a-library.md b/metadata-integration/java/as-a-library.md index dea0a2c94545e7..907b331b8bd964 100644 --- a/metadata-integration/java/as-a-library.md +++ b/metadata-integration/java/as-a-library.md @@ -67,8 +67,8 @@ MetadataChangeProposalWrapper mcpw = MetadataChangeProposalWrapper.builder() .aspect(new DatasetProperties().setDescription("This is the canonical User profile dataset")) .build(); -// Blocking call using future -Future requestFuture = emitter.emit(mcpw, null).get(); +// Blocking call using Future.get() +MetadataWriteResponse requestFuture = emitter.emit(mcpw, null).get(); // Non-blocking using callback emitter.emit(mcpw, new Callback() { From fd2da83ff406877623071c914282843e44e672ab Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Fri, 15 Nov 2024 20:41:21 +0530 Subject: [PATCH 007/174] feat(ingest/cassandra): Add support for Cassandra as a source (#11822) --- .../app/ingest/source/builder/constants.ts | 4 + .../app/ingest/source/builder/sources.json | 7 + .../src/images/cassandralogo.png | Bin 0 -> 127538 bytes .../docs/sources/cassandra/cassandra_pre.md | 40 + .../sources/cassandra/cassandra_recipe.yml | 30 + metadata-ingestion/setup.py | 9 + .../ingestion/source/cassandra/__init__.py | 0 .../ingestion/source/cassandra/cassandra.py | 476 +++ .../source/cassandra/cassandra_api.py | 325 ++ .../source/cassandra/cassandra_config.py | 111 + .../source/cassandra/cassandra_profiling.py | 296 ++ .../source/cassandra/cassandra_utils.py | 152 + .../ingestion/source/common/subtypes.py | 1 + .../ingestion/source/dremio/dremio_config.py | 70 +- .../source/dremio/dremio_profiling.py | 13 +- .../ingestion/source/ge_profiling_config.py | 43 +- .../cassandra/cassandra_mcps_golden.json | 2706 +++++++++++++++++ .../integration/cassandra/docker-compose.yml | 38 + .../cassandra/setup/cassandra.yaml | 1827 +++++++++++ .../cassandra/setup/init_keyspaces.cql | 131 + .../integration/cassandra/test_cassandra.py | 53 + .../tests/unit/test_cassandra_source.py | 81 + .../bootstrap_mcps/data-platforms.yaml | 10 + 23 files changed, 6327 insertions(+), 96 deletions(-) create mode 100644 datahub-web-react/src/images/cassandralogo.png create mode 100644 metadata-ingestion/docs/sources/cassandra/cassandra_pre.md create mode 100644 metadata-ingestion/docs/sources/cassandra/cassandra_recipe.yml create mode 100644 metadata-ingestion/src/datahub/ingestion/source/cassandra/__init__.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_api.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_config.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_profiling.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_utils.py create mode 100644 metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json create mode 100644 metadata-ingestion/tests/integration/cassandra/docker-compose.yml create mode 100644 metadata-ingestion/tests/integration/cassandra/setup/cassandra.yaml create mode 100644 metadata-ingestion/tests/integration/cassandra/setup/init_keyspaces.cql create mode 100644 metadata-ingestion/tests/integration/cassandra/test_cassandra.py create mode 100644 metadata-ingestion/tests/unit/test_cassandra_source.py diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 7b7b880069f676..f892f0ed525d25 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -36,6 +36,7 @@ import csvLogo from '../../../../images/csv-logo.png'; import qlikLogo from '../../../../images/qliklogo.png'; import sigmaLogo from '../../../../images/sigmalogo.png'; import sacLogo from '../../../../images/saclogo.svg'; +import cassandraLogo from '../../../../images/cassandralogo.png'; import datahubLogo from '../../../../images/datahublogo.png'; export const ATHENA = 'athena'; @@ -129,6 +130,8 @@ export const SIGMA = 'sigma'; export const SIGMA_URN = `urn:li:dataPlatform:${SIGMA}`; export const SAC = 'sac'; export const SAC_URN = `urn:li:dataPlatform:${SAC}`; +export const CASSANDRA = 'cassandra'; +export const CASSANDRA_URN = `urn:li:dataPlatform:${CASSANDRA}`; export const DATAHUB = 'datahub'; export const DATAHUB_GC = 'datahub-gc'; export const DATAHUB_LINEAGE_FILE = 'datahub-lineage-file'; @@ -175,6 +178,7 @@ export const PLATFORM_URN_TO_LOGO = { [QLIK_SENSE_URN]: qlikLogo, [SIGMA_URN]: sigmaLogo, [SAC_URN]: sacLogo, + [CASSANDRA_URN]: cassandraLogo, [DATAHUB_URN]: datahubLogo, }; diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 9f54fe23631bcd..c20869a1c849c2 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -310,5 +310,12 @@ "description": "Import Spaces, Sources, Tables and statistics from Dremio.", "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", "recipe": "source:\n type: dremio\n config:\n # Coordinates\n hostname: null\n port: null\n #true if https, otherwise false\n tls: true\n\n #For cloud instance\n #is_dremio_cloud: True\n #dremio_cloud_project_id: \n\n #Credentials with personal access token\n authentication_method: PAT\n password: pass\n\n #Or Credentials with basic auth\n #authentication_method: password\n #username: null\n #password: null\n\n stateful_ingestion:\n enabled: true" + }, + { + "urn": "urn:li:dataPlatform:cassandra", + "name": "cassandra", + "displayName": "CassandraDB", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/cassandra", + "recipe": "source:\n type: cassandra\n config:\n # Credentials for on prem cassandra\n contact_point: localhost\n port: 9042\n username: admin\n password: password\n\n # Or\n # Credentials Astra Cloud\n #cloud_config:\n # secure_connect_bundle: Path to Secure Connect Bundle (.zip)\n # token: Application Token\n\n # Optional Allow / Deny extraction of particular keyspaces.\n keyspace_pattern:\n allow: [.*]\n\n # Optional Allow / Deny extraction of particular tables.\n table_pattern:\n allow: [.*]" } ] diff --git a/datahub-web-react/src/images/cassandralogo.png b/datahub-web-react/src/images/cassandralogo.png new file mode 100644 index 0000000000000000000000000000000000000000..e497dbb6ee76b64af2102bddceaad9b61d74a200 GIT binary patch literal 127538 zcmc$_1zR04*EYKMMmADhi@Ovp?(E`Hytuo&yF-EES{#bIyF+o;7I$}d4t?I|d%tsj z!O2{?va)8$tjSC!$=nH(mla1sz(W8407#M&B8mV2@cKQ=!NIM zfa++(XMLFWI*Fl#q6`4wK?MN#1pxq$?<~K40Kl0U065eG0C-XW035sY7KMMhLB?v5 zCNeUBPwzAw00xK-fO@BZ?*{;g2l!|0odQS!@&A`r1k(Ih1_S_vnggK!E2HsV{!0?? z`Jc}Jlu+5g|7$TD^nav**V$12OUwOJ+Y(L|5bml@spW3Ioa_rF}b?BGP<%d z+B%predgxoW@2VxVqsx;mtb&ow{g;UW3X`~{~sg&$Bu}xqoISjos+q(4aq;d`UbYn zPW)tK{~Y~q`k(7`GB^3Zo@^Zd8`gV(O#jX>eP(24`d{1cs(k;rJaP`^#_yj0*%$cC z_aDjs5AVNz_?Z4V|9@S~|3vy9?t7>L2z*Tcd)ovMKI&|M0RRX3PBY6TZAcvJXYZE8&y{!%GCJe-CG4-!rjP9~F2 zk#H?Wf)!`=NcC5S$w3mjBdjHG%cg_vRkpy{H zsZf^xKzn)IXeNpo9GAL=W@1*;L;+s?P4FFR41Tfym(bfzvTmja^aEN7yJZrc?c?)!wDRro-`e<^M>rO4^>6d5W1pmqxzunm7ZzOK+PdlXa@5Zfjt#9|JJB#i{%ME9oISo9-g55yXv2PT)1{9&t4|B zy29=qUibL(jUwU+xcUiD)fDm}c5Zdm_Fb1dBPbqvxZM4D1S-w~W2`2l`|)vc4vU7- z{Meb+E=>*-E4s?c&Wa>-rC*?lDsq`^mYeEKIc>_)III_&AZ(AEUtfqBblRE-UPEbK zgEpJ*W551ygAahgn-yChX9~REu4m-VPGX&#AZ}%vg)UsF_WEy_@>w(>WDGCE9ht0E z4CzOM$}ge6aZu_`en`eYcAu3T%oMzk=PdVrTB0i?PrGM5li0;zn`rHb94M(x&z=yJ zbcusH;{rzop&owwsJ3TA;MzZ1tlM*fuT!Pht+O`GAF#ZSDWhF{S2rU&^T%a-_F$&d z#J>wbzq-IJ8=JVAccfeU&{8s13{ z*oq2vuHX=r`H#ZCHDa}JTuh&UKt9aROplxD_CAF4)9>=2_A`g@J9Ka;Vl%fvywxf2u>6%A5x>*d~P zqX~F){%Q&lk30ud5Hc2&sn_;L!i)E8tm)6$s8@MCP@QG=BHUjd-7t95(h)(3VjW&C z>y`iFMPo~J`{8icyB$pDGacd0yC>7@s8Rtf)n0y{?~M5O_jh{iIEVQ(|J`tIA4V+5 zv&xRo{^{qx*L9t18)9K$k#&7}=@DHg3v8i& zCM%Tr$wG=6rd5aL_bZ)a%{8b1!$Hj=O9CZ$yw=Pq>hKnFUj@_VClsJ;c_lz zlGxyWSJ#AAE|S4{Xkg;;sAQfaFYx2ZKMDrHs%1Kg>x?`yQn2s`1EOl#pFx%9LpHTA z!FopTI#t+KKL|mMU=!%3ikpgw6qvW^y-%giTRMUtu?fw zKR49seYHymp5?PUU0UI~*2X)YUdFYFZ48;%OmlbIn&h!Z_~fz4nxprO+_WLoO8>eJ z0zm@1byCv7`-3p^-Ea&D+ zXQ>X=?v5R?-5&}h{E7i!yWUe0=)m5;sO=S3&Ri<7J?gcW7wy9*G&qu@4Tg&m!^poC zIlLS#3Th1b65%P5mb||8#15X6L^QZ4AD9c3aBib@rO+%uX@u1`qYn&s|aj!k(XiW?4djv&(i2y8<}>33Z$1Q>f)!8MmA$J_k3m;K%r-tejXp4PbCusVZrtC>ohYY1)e*FLn*a((8icsS+NDR^^Yg&Dr{(jw40@12o$lk`n=d4sOq>+90-*& z`SwZhrQhER|ITGQ&TSWyFvNyP79P5PYu9DMZ*PUvrx-=LdeSj}i+@bxNTn&7 zQ;<`8G#Y)lJzVtxU`=BsNbisa7sd-vA}84l7KP|t1gKUI3y5Sts(fU8rKLd?1R3ED zBjNhG90-`%UI^R-7YC7*O|KaTpx^xGyh$zN=28ypBD;V92ZYXQi{2T9BF$p!9eS)L6XM?*e4_gF)k+D-!s_^oPn|I3_1IkrR zYQ#kBPiHbZc0lX0SJu{Ep~F&EmMwY&wL z;SMKSwbk$zE`NY0Y= z*Fw#bTLO59wT|y*wHexqM&!7n67d47Y?xTRpUkW$)d~Mz-zM_ky9dyOtQe^`Ah;W^ zcf%SfcCZt9Xn7c(7CLvzJpwa@Hkgg_H!T67^Frx|8`hgI`zc!`;bLb7_=$zP5=d8d z(?PD!PUw?s7veD#>)L((ZgwBxm_hK|+SbBdaDYXSiGOY@5d!opKI*aT`5*k2M)Iva z^a6B)ro$%|sz)%qB}#&8ErTLA3=E76G)KUMx=<5Er7S$GV8l66{1+9nKRY#P8-y7*EuFZ4TlK?dJoKqFAy>mxeugf%q@jll`I zJTH%!&f|5J#^*;yel=(%ztoIcPH-Yr%G9W{vBaPMe zK*7kGZky4%oO%_2rV&wcQE0#|CT2Rdypzu_7V~J_)0B!W@(jM<9EE2+_7=e$J#3-j zELbpj!|(_cLE*kQlG^?CWN^%UJDk^X1OSkUv2c! zxgrm>q>DSzhYBAzb(=a~8=DrNrpA?ja&z09?S!i$4V1n^@Wb?&jC56E97B&huJSBFv;|HX&~{!xE9L78mmPc#E49`D`JQj? z?myq)nr=iFMcw4>3N2yDNwTy>}#9K8MT^dys@m7Qc|2QcEvHQV6-cbhhTvn7cYJK8??XG-@{p+_QtF-KHj7km5x#5 z3qPY}+1CmprRXAPp9)|Gp*}5D#o7A2+$@yRNbuQSeQZ=$f5O@N63IR{=ZM>xHFVzn zh3D5+Y+-_Pv(H7}4HO6tc-1-e7LrrO9Dq?F878v(osX%4V)adfgq!FomgvK3v?2#G z-iFk+z|PI1jOJY{8K>>)@%abJ8~pR3kd-R@pWoQ86TyX+#O4_P^dN?B|ILOM4;X@npCGzE}XJ=$>vNn*|dLBsV;S=8S z7VDQ{`l)h!`S(cjo93RWb64OxT`_CH*QqC$h||EN`lsPxqDdc5Hkyf6QcB><$~k7d zH5Vh-Bx>ZYWEJx%r!+J?CLk2WQqXjo0$u?AhH41IB$l;my!nfbg?8odxu~$sINpsD zYMbYpTxgpWI6EyUyTOy=E}4&C!cP{*!!sg4hZp7wPtwxH@G_8nZWzb|dfajb7d&W1 zp2cTDUbhoH+Y`b6sc2WO)*B{U=L>IE_UZ)G@(W>%I9UZfT)dY4{c3EJLQ;{bekp?X zOz^x9NEL@tF0%-tA_X1(+nff)S2Y9BuXY%(D%7UniO)qn(GvmPGq%yhFDjv~by?Q= z{V-emEwJ^TX_644TttQ|>wcNLlTv#UJbi(_?&KnD_$$*eroW~m zI=o7-D^I+Qg&?r)rBB9k596w?6@}1d(BH*7j_L*CjYwqO9k`ME)(+A*(mhEePC41h ze%3TQ8fZ7&G{AMcKh-=vQQuSa%fp-u6@>NefbtH1ne^NW;Yv4Ni2@71{J#3`6gqTw z^#Ny-5>E@bE!louz8l{Y(7?a;1FauAe{6k-`jg z7Cxw)%sAsfy_EToF|H5PD?hCUTnZi@f&|uUHjQPx6}m zmCta`{?gWs(^k{-=h~RHYy-oPKlwN?NENj>0~n{{w@*O6(bo|5dBg^;l}TyCStlQ) zMXkMan>m(1*b@Rbgv&-Wd!No{@{z@* z@_Vd|REzhd|NKRY50Ab8mO~ z^Bm(2o~4~{r?FL)6_i-80y^=fOMdX;{iy-TW`nX0@;qyHdcL&H^Tn*Y>QOt=p z_rAiWm+=qY1OYN&BTpQ+4C|CuK}h@(2j6Mc6|DdrdSd;8+=Y22`x<--DzGH#mTt%< zd@tA5buYACla1SH+mUOtJyGBUMhybFZY4KD4~QIe%xw4<&V6jT1M}%D4VseQN{T{Q z6k+Hd*eH7n!?V2#Pm>XfJJrq3ufvJ0eQEQUbd|5LJ2_0xiF^FtpGtt6^esE3qaWWQ zJ}uN*#KN8B0p9+yTOQW&(NA0;FMf2t^9Y+^01ZACq%nkrM?@`WMfPwB#X<2yeBAZ#Y zKb9Sj;+z5w z(2ESyN&LyftQN|?fw6P(^hG0=X_Of-rRopm)wD&9sO!E1(>(P+i6CprZ^_t+viS)P zz!m-V4MkugZW`hpW)H1D-kUvWi_N5d3FD6pq4OYlJK2UR9A^$9Jvv7|p-#wu#^aRp zUqE5;v7ZU}Q)s5`tDqw|w0TqZRp!Yf;L5j`^;`+%em0ZJ`RR^*&7+RPL#kWL?~)#z z(5;u`m&DT9%dZWKlO;PAm5o-J+qBy)c%$j>j(4J^t)ar9+)J0vo0mb+-TsGMy)e)P+iw)TNs z3+1H$tVmCaA_GW}f2MG1`Y}IbLnDX|@!YMI$3P?X*rMv)B1qflc=G5HK#dAUUF#nE zHv=e57{DNhG|k_uCvPtio$dE`?vztFj<8sai`C^G53Cc6TDO&8->Xk4_q6)N>!kW? zm?_-(GyIQ6VhapD`3vKbydjKPcS396j;G+vIq+bIXdHxC-90al>slVH3RguYpoU0-7?3@t~LZ8usiK51M>p={#>nhVcg=URS`^K zvWewt5{mJ4Lk0DkC#iQtvkKPZ!FdQCbaBV4m~@%08x+>lf#wk zh&oEmSKtTJ`gn2dV%wiblav77MKxWFeO`mI$e;EN`T2!Q<1+YGJ2TOKn%c0qeJnTC@6u0bH4}=*P{D9rUZO+%x|t^);}fzn(w-qq3(9sY z@NNwk+-Jv@D*={nDR&?-O&k;=zH|gFJzavjgyKkw7LoXxY`tX|8C|mM%BWk554w;i zG_6Tr8E9|UjzC{8fvOYHG+iK6CP)t0}Ba> z3=o@#P%_ zY2a)%GVqSgh$q5(b_<@ryPGdoGUvzFk+Y@FYRyW49kpD|J8bJkwU*kVPbo>U;=SOm z4Ky<*cR&m#_1%__U{%Ef>t<|r)rODh@$@*z;%9oP@>Tpjxm77yD)-2Fl=R0 zj(6$U*|SyYUIG*zm=_mE7wN_Y?MK!L63YAN4Nd2(+RL)FBnO_54lxnij!zI%PgrkttriJyfoaQ5FU0aJ0306dBdd^EG%7AN5>Y9jV_so z{NF^7M`62|#JJ92Vmz5#ZM5|dNI`b0Y>N?my@!X^=RJiwxDSO(9i`F5sOJ5cCG!BD zetCEwU?@|S2CU-_j227F9UQXY%LfsHd>)8_l;#L3Zb5vKo80-`+lZtFS zANB1QZRPwEwOTUViprDNIK1Q7zYM;LCMpd|2ImS~0Gvu72{1Hw>%mt-FJlW43V6fq z&Ogn-eqD(3_A<&Nch7+UsmtUCXcamxHCQ!&q|-H=8lUOucU?HbtVMysGoUS8)ctJp z%3_ZSa-?ibY8svbesQXmNAN+}M~<%ndNE^2kaaHBO(Qw5R@WNheIpt=`gh8%ojY!rGC+ERJw&N0Wc*Ao;?QDqq!D z-6LP2=)1;g|zUVrI{$)L4i1n7*>(YPsn zgmJ=)028d-v&62f?YaA)FkdA`-4F=+Z^|=`WN3=C3e&BuNrks&wl}Xg0@jB=!F-c~ z5ry@savF*o>u{Ks;%3Oa#Y9x!1TMZ%oCpj+Id+aA1Sz$3XR*}Qo~e>f`h34UO5Q48 zzl9DDnWskGFC}1r%NxAs#iBo)a{mtgYW%dbUU3FL^GX&R?64eoKHS8v@2=yjb4S>l zr~Ht|W`6qiv>*Q_BIyxP8^=8@^q{1kC$t@w1;1*R!38%6uwHEniFbJpvZ9{R+t`TkdF zfMa5g^9d(L(F4ZmO(&e2#1h7Ve%YbIA}6$H&(weqoT%&GSUn!M~N|7TBhHkv328isJQL=xcp*1@byQiFf$RgxeOxK zGvS>y6!-U&e}P_~`3dIF&$W76Un~hf_HK3kTQOz}TD2@Q`wgPgsb*5x5qUiYC10I9>%gjrKEmguqY&yPG_s_uxJ<24 z{~;I0=WcGKwiSiw&YbFRHCIe83i0fwLwpAY>89opGOCHbAWXjMhK25djDlli*W5Br z-4oDz+i}~u!sD#NDR(_<`|e}eogxb{D15A$#8Pe#6B_I$G1L^=;yg@S5R)r$7CC$K zrN3^$p<{)`xi*XM+{xHoH*6-<7=7u4j_K**Rcl0uJNo{P%4WwvWwg6Eo;vyqsYjR# z4y17SV&z5yOLqRz6nCq0Vj@c|q>}51$4cyzuyMpgQC$S&MlXtm>#b; zj6**XX>LJw@HxdS>F>9luE=t7fZM4uk8&ODWu#SqN6nlug8}w9eGTp>i(Ob-{gO27 z)ws~>OJicH%6~@}hxh-gtvky?BD=*~IyW%b-|Hl<{id~xzzbljJaj+2;Vy8Mxx zKnli98m_DLo5Ek~=Cqf`Ha<`zoXrZ89{lP_AU8D@IZ2FVUWaAXEwNswl3pg9fZsz_ zHS^7os()b{S`Wg-7rsf)n_hYAlL;N~EhGlyJj>h>%Eom>y(ltPWKd=(wZ_W?*m1|+ zl0olx2i$%JPHU;LhIiL2J$k@_&0G}%)h&I-9Y)3`dRg4!-nnHD{bkjQL-0agpU8 z0e`N$s3528pThyk#vK8x_PkwM{d-!jaHGJb_}EQVfPku@0yansk6MLTiQ8t1G%{;T z;5D)<{o!g1r;j@*BFs1MpsCGaQd|lRhL#*!(B{$6+&d01*he&Uk=2q}&)8LI6P6-T228Af0?H2;SmnER1 z#VkOzzAuqk9<(A*#gkQKRvR{IF(76a3B!NT_zL3FIz}F=`sNE~=c_6OlJxynfj?6) zYCO8rA#Au<3)J-~Pa5LCps<_H{ag~NPwo9>!uvAugFp%|7ItH^*7n47=98MYtz#R& zc7RSi>4DY^dqOEnh3YG`-htiHH=Hs1a`&P$2vDv(|Kck>= z;*ANkkpb%-8N{BW&6PjuwbI#?^4s=nCJPe@>#jZxmy9oWHo75@=*YJ?wp#lb2A8`Z zeAK*cc_MN&LKUj8TfEj2v$+GD@>&*^n>^Qz%bYo=a1yhb6ZS>hV8rJWg9&0d`^kR= zzCdPr2TKl)MxW#)?KL5Cn!umSZX?tbDgTsm~@?Q;Ep9en!SiM&^5_&uI6Odykf%=;UvMVi&#YJ9nTI9pi zOg-j?Hy$TH62rr`CGu{#g(~XO5%a0mVrMiDvc2S!Uzh>&j|jx#R!(F1D$)3}3f~HG z^}pb4B(vfVC5lmFD*Mj z_g=u_{3=^?Wkd!!AR3XnBEQmzYRbPoHUc>zNlQ(m&G5NHn-?T%sZ)y-p<6R{wYz9j z(09F)sk0?3HFi3coLHc)MMrS1#7qPCq;QPQydK3rQzch5h=|KpD(#0 znp=`9tN(Epa@FBLtNtl_?hKXW4)3sw22VnpR}az|hG4bTb{gS0JW?q0OlR{CG&U|J z=ZjhTX3!-(Uv~OFCl@#w@%B|B!ECYK=m~aUA_T<^&+t9yMR~U2W8Vk@3uoGt7ciT4 zhmG-TUwbzqkJkd~`A~X2Sf478)&EdZQl@fG56*TVe$F5UFSBR>9P5=w@P1{8GyM+t z&ZSJQL)m>tlYEjugw+)V;wI|t@a_cP`3j!KBjPY>Hss~A6zzSdt|TgyFxMwtg%@4g z6H2>3NpcRZuuP)YynX6x6OW0Rf0T^>^c*_BZFKn$R~ZiFxF;4!*}hPKI!gsyN0e6Q zTftHOqYl1P)orqsyLia|y(Er}V_jE|xVLyHC}<$ED{}ccrX_=oB{efXx#!*#fjH7> zoq>}F6tY<=YE6&i&OXZ$lQ=TkV%yHZG@ks_jcdPp)(%Eq-#d z73U*2SRg{ySv&SL?##xX#)6l!O&JjtfteAXV4-v4Ga(F(m&?2^qrAe2Ut>x+L%Q?I z#`_tQsb_1jT(%3Og04CXQ7V`2`Io?}N}=9~*{m?M7;u2F$;;!|xDB*}n>} z<5Z{F*W3qgjIp6y-0RVY62tm@(h-7tMx>h83D+WX0BFotygS0UBt zgSCC@Dsd&zsc9nC&N}fP0Sznik4Bbt$j5xKJ+F@`O~g#dJ{N*M<%gy<)kJLZ>t@(sv1xIPr;G;J8 z$+F`IrU4#}Bdo((w(f*cdd+H&p@?BBX5Za%Gml+3#H?PP>!$WVUYBDDW4@(wnqB|x z$|`79O@QW_gAZZpL$b7?UWLJp-;&ftz9d#tzeRus&_)k&hZCTs(I#qhku_DSqB=WY zaf^sOXpY5A$eM912C;5`%vU4V4xuIaY%ALt4OQRnv&<4}x)p05Q77ik!!_on_?b~c7f9WFXqW{N zh@f=2B{pX2Nz1Jv?d1);@cY`{(?d_lX;5v?^ikCuL3&B3lOy#0F&#z2+44P=LIz3ymnJFH-+tii z``(rYXir_mBy@fg{zOLxsPR@gsFfrLgB|?fvZAI@hpVfyZ1IjABUAMWAe0r$NaDrG zHEvUK(xQJ2OR0$l7KlZR3&4mP4YdB1KM1wI8Ivsxz#XoW@IYQ4N@8Cj&vC+jFuC_v zo>$w>U>BkUdHo#Z2kAGOfU_J8!*#eAX7E^R%KdfOu()M34Tb|OjgS9iY^hlb+rwoi z4A#g^3G6Atttb+^ zWOp}VS?Xj)Y?f1H=o5Z%Gb^8+Q%yC@Qh$5j!rzEOF#oh_e&FA!&Bn6cwFJf14vzdG zjFPY42-;RHifnRD(`>V0 zV_X^(y}yd)jkzD%^tRx*%5X~_5G|4#`Du)a?A2dlS=KgmreQT-rQD8B)0e=on^}g^ev@WRB8WALt#NQ)%F4B?{HK~w6 zm1-&^8sjrUHNBb8n?`^BR&^7PRMNai6jwDQ6_1Mw?FN$*4bQ7F&hku{ZF@}~c_qR! z3t|4j4$NWMi3U5B4)(+3pId%yvB63`UU{N=jO)p{8+7=&V<(6o<}huyAZaeB;U}0d zw1{Xc(%e#?`y|YGTR(HQv_9VwC!P_Kjt+@1UWC@t`uS)$HybCMG}L)7(btzizh+xy zyETaj2*Q`46(Dy#h&9tW_D$yWOPTeXS~9#RqL_j<8=mAt@(0LS@{O#G`ast$z1LjF z<0&l-?&@SmJJVWhA?3NY@r2ky8~TX4s@ zC8Q>WZxzF;IXEnd9HAG-jmT9g%5A5$?gjtF-U4`Ny{*~*=h>lu*H!93o=QEp5XvEC zTnhdd2UBSdgje%t#z)ku zA-;Bb4Pk9O){oZ_8&fYd0{lOzsC6EsLLSorFXO2#EoU9Ck4MV0#I_3?({U0|6Gy{( z!oAXnDizC8%^km24Y0`m^MI5PK#E{1nn0+&WwhUuMSr=!>C~TW653Y0L2qr;x>&Eq zRPWUPR%!e5kpO8$Ec(ap^`{R7AwsmNiUr@Dzz7;bz3sf>fmXvVwxYGFy+G(oVLyuG zn(bnU_!IImV3dT>$Vb0RWBm|-O@ahfqA(5U_!t06;(MI?pb3_lhkC~WqT_!*MOwS%7eEHA!_TMqm(yo#elI(v9_?sjDC zWSJ~CWm;0Nvp>2*S-o1!lsPgr+IBj^Hz^`mgtiFV+Q}W}=OU$;zY289SqdxC*~(H4 z!B*sB(Odw?lpDEZ41|iT&l(f+IlppbcJ_}fm|3W8O4 z>s$`pzr`;8xli2nn}r??TNe*|XiI=PRc=+*X=|VK#OZo?;WThuTY_eh=FP7Poj>evmEQX<<;6PM4h;G7DcS6kc|)molbA4fno z^bYlwPI=Y`O8`f5?CSNQBKofCvC@8KneTl#G~WVT5^;b0k!5%f5q6d-w_a+QOR7}U zXX?DJv0l@*5Qe?3KE95Dx-3e7>aRa)y{m3HZ7&x*`?l|iuLuS$CiH;*W$l_(ixGA+3j_2nm(S7ujAt~oe2vQ_kfakm)J8r(T~oPov^Z3 z5jayM4AS8BJjcMe0ItCLecu4%N6H&hxr3OwRmJz#9WF6{2VH&2=8X2b*amkUKqK_< z(^!EVeX+P~`Gz9fEu~f+Cms3gB-|CWL+g}x2xQ|Naw_Z0ce_Vvm0;13?g4G7STjrp z5qIz1nak&3Ta2Wq`nn_aapKRZd^)XrftSv#-lT12Lvep~2+4BB8XB)joRr=-e@z-A z7b%gM1?m7o>Gu48A9>hlbZl=z<>zNHt~B4I^QUqCnQwlpk zVcIke@s&~O7yb?MRC!lY*Leh+2k-Y4ef{@BHuDpV+{9D&Uynl9+D6xFr4}#X>GoG z9VL}Fm`w>iEp+NuUE8P~!#rU3(X3M9ls?qC z*1EQb)tNb<#5SShq;-*87XGg5h?-yyZ9fM?8RZ=qXwj-tga6^>7;gg<+qAT z&A{;Oam{dChMz}4a81>I>mJm#7dw`;&I{u*iJvJyK_mhXW?$=Zn;1~^ z3P>nB+a)2uj}Jw~^NBG1c?Ju?c`)`gevV_ov}W2ZnxXszP0$}-NpQR~ibS7%Rm7Pw z&i^dJytl=4#{%msgQ#~($8_Xgu|~k>bzgh(^(sPj^>2nrTMMSn3LQ=>PHW6yX$m+n z`l8$K484wL53Y6bhCcC{dpYd@0Rz|IF-G=YD)DwnN=B!?frYB_{gEL8&mPQIXqVDx zHy3KPxZGOLgvq|S&G|zSmoTmgx~3O~{y2U3BA^nfzpJ?+-omR$borTx zEhKH(AIe{i5$zgF(y)xXwl$d50U%4S18G&jcIBDpTu1Ncp+Kch%IXxU-cEXtNjk4$I+Jw4_ zk1hUeYb#8Ni-tDE+##Zvrgmkw%2IiuW1)DC>TrOMPir13SH4i9!8?Uyo>yX3=ioH+mYnEy?rh_-_yp@kZ z_EaI7eB;^eQeLnP$xQD2L;mwz>%PhA8B^_6s$-cM3N;ohp?LU*6&6L6{FT)OJZiyD z@n+X{8a6uQ6%(k8-vY8Ig^l6~!fYfyR^V3B10y1ul5WN(nn!>Zp#E#iF6GsfIi&M` znel5{|Hdw6*w@qc@W4W-$)I`8k0`f(60S6#pH_sjpAQidLax8H>O1Ag5X~Y+@D1`^ zib{8BaC0o_*%H|_q?eLS?p-7@bm=(bM&1b%{Y7z#OXR=$0sY3t7Y2S4NTb>3{iwLc=^79>@X!dxC z{eo%&u~CwLLJ(nt_ZI9DcyNJm*|IrUIP$YK==`T9sRtjAp%I6ERI3l$E5tD`r-2HF zr>Awd)59?KZ*QIJlFT=bG?Jqro|iNG5n~$-A1u8W%B^YEY9D8*=oW6&C&Oj-1b6&k zYd^u?fH`S^JJrl_N5I)S52UdwlI2@NRv%_K#QnRN@4%||kC#jCU^ThRJxA$_wFcvn zEx-+B0^O$3!ISo+diQcAtz{$H^&J4Ao9gTl&G5189W&N=dJ(xS%*&3GRkHjeO%1vz zO5z)VW(^0r`wx=jrQ|ZLWw3&Y5tT1aM zNuBqdHK@p3s0->7NV_am4Hq<|!uRh+1dnO4uYa+F+(}^NNG@)9uNv z>U@>=v#lKJb*2r>N-e{z+oZ)bVwulxv@4rlx?pn7YYs3gikr+M4*(u z?LMic8qvB^!Xo$v{PmGd5e`A$^O}Q|;T0V+IUx?N`&*<}n|L8ZJ{Rl`%dFf3pbJ0m zqx6UCzcoy@;ZVL(-=qVfCvF&53|e1bW3Xk~`FPG^235zq*71u(&0w)D$oDHuN}@pieMNdxaQG{f%^bYqo|8Ecq< zd*<9L3<*CYTeZBo=ee3647bick-9`>kjLz6UZ?dhYG5ZxSJyIj4u?c%)wR6+v0u<( zVSA%gX7)T?UQgA1<0)Z}efU;YriEccVo$kns@Nd>tRr6Jkjnss0djs_3UmC{2JXsLS_IQS)N&3pnvG(Xl ze4Ka&twSx6@y4c0RoMVRe1|%bu%UJZbh4s`QShM2`jDlwZKUub^yR zoK|1IAe@s^n>bX_ky4Lj59~;N$vh`E2T~9idNCy=Tt}9six52@q4qgmgb(mygm^I| z1i++#v~4F$Bp41WQH&450@K~BiK_qrKmbWZK~$LajXc1Dt7R%zC=G>L5#O4!mD|JW z0dLKnmdjl3TM^-4k$c_!B1Yx7Hz0O3wy*Q**ua4U-JE@SWcJhn zMo^_<#s9=Un7A*()lJ>c0X=dWFH2orgOwhvw1WcpUE6KX*3DMBZIe~)+pSv|gYQPL zU5-=Cbj+eBkRBUmbgc3#p!{YmUoDra_Uds9oID1{a>MhOTRalkN~YUUOb;@|a1vJs@Vk)v^|@2} zeMA7ON7NkPVg)t*+ejJuirPys^5=~4&!@ka2(Xk*fq7?4oq7`f{U>=JMv_k+16}iJ zb6o)?OA7xVdB^53b#oC@+W3-j>=ClI1?dt7i~S+v&viK6&3oc<*kQT zKjLNnH(%7-!}-Gob17f`@FanO4}{Xvt^~}lNezdeV>O9`@a$k`>+zY{Wkw}${Ey&3 z3V|b`jT<-WC3G(yFT?;|h|4(mG}`tg39y70L!DiH>+t%Q1M!$FaSN8XIoDF^AtF7O zDDR#Ys5xk-oN|h7NG-588UognNah9!X9wJ#g)87z;(U{>4s^}^$m<`ogIUBYn`(6c zL$8bs%OntPX)Sqi_ezP-qAczQxVmy{BX-A`!|m#`##&wm?t9hm)yJX`_N%+@vfF?B zV;eVdvh_-e1)h?qQQ$#5#Th2MD_5Z4h_Vx<3M@?f~~z0KIQ0)#h9IJc5dgJ>6>idwvASE zfVuQJy(}}Yrv>SM>Q9=Tp|N|{62IRyudB^t+}0>Dr(Qzv-?)=);OL2#*SnBd%SdW| zL8?_9v~5}0cI~u`zzbNt7twvLCpi8zwF^LO4V+Y?nkTZ9(sO$&}ti@dC6JXVN#>dR$({m$y77F(Fdw(?S&m>wi1wi_Cpiw+;M|ct<@vU zHJ@Bo&hyVbXV-oCOSXRFCcEH0=UL^hHFluSxpr{eS+=uBf9sb<;9WyKOT9)c4c98p zfEJVLlaw8RCB37f-iG$dwX=uxv?q5QU~!e8)$$yar?sSO(np4bQf%MBTDx-E5F1Va z-lD1o>!Cos!;?>~RojX4Em04_{0G_`{3t%5tbr2@ZHyUGXgJsH@yhPjBeK)FcN<~5 zc9+=i@BOX);;vuWgz@8TYw=#&w7<$~sw%CH>1$!FV?ZKU+<8qT(48^{?;OCpiztIG z7k@afVUL=6d14ir9<;oIzLwv+(1wheV71j%c5q**mF(JPyNfp3?zh%hS8D+Z_hA`40NcL%&IxrFN;G(thGoUHCfOb)S3nxzOHL z0c`p<`35S!8$ensFc)B9eL9A-H0-yi>^9W}m`fv11ap*eOm!*egzRrmMvMb(aR_v zcR6{AP94fce}*6anx2+6gTT(asN)5CWJ6cMTm&7a@a=M&k~bdT97vfpYgUR@aPnM2 z0W9)Mc#Rw}uqG4O`G5zM=0 z&UFss;93byi?n#-=b2btZ&7Vs`F`uyub(SJpywjJRi^vNN{`6|7xX34QSQrk> z4<0;dci(-tedjyhVfohS_P&| zq`$4*=Q1?=qt=T?)%i9jnq@kohkypNV#PYnA5L=ei*a%R zc34IT@Al0+-}k6ceNm2$+S(%i=~3hQ=QHiAAEjE6LG?T&+KC z>eT6IP>(S*F{rM-{y~O4{vM>hh0B+(_X>^)txnd`VuLzaMl$=mg9Cn)tU6MJP8VVP zt2@!upX5dR9KHBt2|URF56IQ(#Fw*OE=Z6Gz@7V3pv$WI*H}|;zkPGB zjp?2NXhi_8mZgTPg=-A}+c=hswH}#PgmtfR^wuPRnkbk2V65}i)OY1c1$I)u9#)1j z!sjKdf%c|5B`Ma1mfGo`Q&}>vigDW)aIYk$&K>O}wBeCBW#*DWJ(xg{VsCBT;htH= zu#*|+yv`yo-UYvB+8uY?f!lq6eeG-4ThGjpRTJZcThqGt=;gq)5f6%saA&T1B`)-e z1FVk^$m<-F!Cik$p?t+hA=vSpU|9;$Rlg&GEj*K3U1%t94>Fg%XWxGAH*EYQt2*mF zZrpu49~0WS{&g$gwGqp`m!-3gKq}y_vIOuBWF)b*4x@={z0rp41I*y z*?qGux8JC=u|u;XH(7^nNbeP`1>=KDJ^S6{i4ysxH3BC8k2mUb1dT6@i!meTBKmKRm; z=$ba|qZBeH6bfYl=1(A0A4QRgE?luS@fJQ zS?Wu7+qtvOv9EvqYc_4lBs=r$S+=v1PtQgL&blq;I?rNABs(Y93i=kx^=ftWNr8IB zK>(g#4f~3ir=xKPt>=CLw*z#Zkw|gRPepPpt6D!*Aq7=Sy5<%1wjpCDTIJdAvAsLC z+0KpYZ5M9&@?Aw1WaYYaMa?tju0G?|VxVn_W$JwiGA=*(j5FPHidE{G9ILOP9_$G; zcVCO5qn+Yo%MgDDmF?#JUVy83KSn?yUnRQND7QZBDoTkq+ zxeK~pVPJp#D=Sx)a9;Pe_F8hO^&}TNvNh)RpEm7B@Z~$urOzS1LLCn>UKy#bt9z9a z)aEYNBMLk-MZ3xLpT+@ov@d<>OIq&G0Xc5%VWZ(u;{w~RAg1V6d~noI?a3mWwo zb+~qugp`C6k(=(PF{^_m#ddJ_Hr$kXHek#|d(S18*@SUpZPlWe%zBTo{UfI{?>fVX z@^paf9Nz*x$pV!p;i~a(^?O;^tm2l7KrDLXxAyIuZnC<}KK9Z9i0;w0AlUsYw_E@0knN|emNay6tS?H+YFy8gS>)zh=Z<&F$yRdwlNDPmjk+{G zEr}y;G*5cq3YZ&#oBu7w-@h}lzl|x#wP!b#+CVH|U2Cb{=H(pDF(VUJ3Z43(`0W== zLNThcn;x5MTXGAmC+F(azj_n`lq?Rjb)f}wdfSR+OYNeIE@U;h?sl*S&y4`tOTI$r z_w6*M5N7A*T7J(0>m^IOcVFDxJy>LhAr|V^Oo}>wS}InFPhdNM_xa`b3q4yUmFobW z(y_74IDMv2GQ?(3>@M5=D|HXoaK%`J{B$Sx#cwlQ{^c+I;m6R@NSt}ZTTsh z{vAr1!lkk0VraV+Iya)N&trB$MNVesm+??NjO;jzcL5PVLs|#(RDgCNZ?b^1k}?*o zSh2#7>h1`bD^NFW|Ni}vl9G}}BG{uS6N3f~81OX)zwd*F_fkK`wF?jwGxa+8kEns5i z0L)BY=e?FYufx;}$OUSwa>R=fwTk`wSQ+k3%O7}>oiyb%8-h?P95{&2g!ZxmK2GxT zlYg<|+EnW?a;nucwjDxvy*&o5`N{Nes*fV{v;?cxP4`{sl^Iz!u(8Hgkwu^3JYz&5 zp(|S~D?7+)VF03D&N$_5w?S`Fcd}H=D~ANX=Cl#cs`bGmkeSGss!(&SDo$KE1Bgcl3`M#;eJ)b>Z+@+Wvp= zZy!Gv+>c(dt*Gd0DDPvW$VhgOSC(*doz#I93IRHMcpe4bqk(+1gC+BqFW*Q0b^$GM z`k5c7%XGbQyz8HZrYiyRo{D4fXpkqdBN!J4bI7@eHobuIy%QJs8{Dtd>Fy1Xhcs1> zUj+KjCgAS@z^r1eUJ6M^6OT{H@ic)qvgyDu!x61-36^F5Kcdhr7H?uW$TUV4HZade@aPMJ(nT?@TeyB}*CI z^YU!stPAXkwfiiKZ-DRDSS;t8C)s*=GXDVKB^!(F%CpAV_oofAA1vEuWAZXkE}GsU ztniE9I35i=LEAU;t>Q$``T*3$8paTPFo?h++c{M6B$)3?8A)6YK7atDvJaugaVc`c>p z{RIK+*5g;X`qEp^o;@te>-sCB_w3 zw(13F8WdEgGV$W{88pNS1`M)Mr%YoR+V>eBEV8247TJLBiethCtFQuqD!u z**V(58bIOK0i_4f+E1n(lkqr>#3~=cu|QOsl^EQTTib!AlzlAoCov%YK*mr?D71L~ zqD8y7rzJph4{J7M&hvtFJ-51B6lUwztrt+v)!cglIzo1WGNH0sqJh(gB)2l1&T{`R zsqkfF&;BLLmM!vfiiXZ~@Az`Ey8qN^)BYbf|3G6Sq7UA!;rOTUt~S`%xNO0K1zPXE zrMBz#PNyWGcba7<+zo+Qvu33N;_)#iEb`NM5xzmkx|A2ebu>?-6})sXQdZWEqJy=t z5}<%ktwzArF*aT}YP?N8^J*J3aulx1K9<24xrCV9hYCp2ejWK&EO^PfzxN|nmD`g~ zXx1`q3@cpgsAuH3kD)j~$Eu0R2>qo_dGP=dy<>a@+bX1coPCZhTe{RP_{hiYiz9p6 z-RpPT(3~{e--5N;+SAiptA$mDFpX@>x^;ZeW4a5hYAt_D=T(Mp(VxZI?Y3fir(a#5X_WUjbnWU&Y|EO)$I=TZU{EkAJN80$B5gpE7(EGyo!(Td(! zVY}8Y#~Yu{cWHXW0_k+0TOXqJkbkeU`3XF%$ADU~VCmAWovefGZKoulx6Pju`RXCTZf>eXAaDbXx>PTMTvK#5 zy@+}VJM<#xrIw3PKUpzVxF2hALw4)kA7OO1jTk!~m-=AmuG9;xcU>J?`%0=)7SL{K&{9IM(zugG+dX&Q zXy3l^X8Yj8L3a0w?Uvs?-F5*2?$f2Q8{7P=vFIww?5(e@@cr?kO`B}&_=#=a)6zvR zT*~q@d9`S(TJ4`}DZi#wgWUGVy_I(Jo&&hg^X!+W4Yh0MzGofaxvsd?5+i1qL6@l(cMoHK@fj|G>KVhI#EE>7)Pg%=BY4DfgYEGXdXdg6@mALcI z8I))LbKV4(<)3fv!H3V>cQQzTh_0iRB&7?WVH0!jzesfcp!_U*bKM%d@0GPS(em8%^zoKnFIG)fWDE}OIE3@cjv*O&*~j=G>8uN^&Tn7 zyOqbsVe$(4m?Gs%cfHq&)~>RxuP?KMyEX&tc(|A*rzvn!pao=|ZHueuF#b><+M~L* z3#If|k`wT?CO_0zeXNC0i|E~e+I(bB9%JSUS#j+xm4&epV}K64geC6w0LE^C#>O#N z&F|yH1^lMt5kR-{0(IY(U+RjTd9W_94|!C_O+Nr z6ahs{$&#=^h^Y<W8gx(0Ha)QNHrZ`er{C_}2X0 zY<{daE=oX_+0Y)nP0=))-g?WXojlbnKf`w5V(y6`<*O4Ne#_+(uf{ueh27R%ot@qdVQyDU@BX0 ztYZZ;D6fi+QNdz92vnD#O$*&zhG*hc=A>t2HJyMEqW&#_*!uwd$wToPko>l2o6S#v z-aJW~=~pMqm|A~)!9=3!(`+M3$3uJgG8r`UjlAZ!9noX8Rc&|fe%4I6bmCxFJ{Ax@ z_&%18+j=A4kk{Byb4t>9j@t*0vy(^8vc*sQ(N->AWakYW(rVyeM+i?t`H>4NF{y9J z(pHj^oXfCMw;lyVF!SM;@^UP9z#V|Esl;;E6wSmjcPUzw_jZw5q$y@->OC1Sf_Rf5 z!)@&JnYMG&2CVj#wr9giOJSHPr)Qyim$X>Uu>^M78)r3itBbUilvghhAs^1gzAOru zBST)n)x0(FMp@~6*FKw}xPHvf9*Xia8ql7B0x*TxgW-}}s#?YYE_*qy^eDu-4#=;> znp{_Li{Sv~B^3H&+y|Qkv#DumFFYGtK&wMwjt3!S@k0-Vo?X7YPJI%CX%dRTk1=-6 z^JJ_)8{kFixf?3dkO-^EyPkc~t`z5I?C|jZc9t23S6lM#qs;*~g@X&c12%ljcF~b%l)}|`PPkJ@BV{KUtW9b0Dv@lJjQ7X?spcG!BNc1pkhe=Hg?q3U%6cC zu<(`8_-*QpQ|*Tn2iX5E*z6V{SqHGo4c(EqM#gysWEBn>U{5^w3;X<+uCu(nSlMmS zka*1p%WXS~(STxsSg0zAS(@xKqRNL|L}1ibv99_BrwnsJYj>SF$}aoM3hP&pWowz2 z?t2=17WpoBjiobcVyEE>m)rdS*0(@f!D@n;Y>6f!ygvy~a&cMvLjsNns#Vmyn3G7M zZmHnhNCm>}tF}LUV1iBYri#5)%qO@j>#dOS^GfPCno0B*(ze$aE*M1M-?R~|mPTKB z;kkdAjUCP0Y%FKdfHIhv)Q>b%qAoMXN{39f(HVcVm;e2Y&3Ny{M3ZN;l3?44R$^KH zzFMl-1pRm%4o&>l`_Ej@(6_dkk!9U`6kJo+GC6|3i*uB%gK)>KvX!Ij>NXVU}4sGDSntPLqE`XRd{xAdT$ z5fU$$6CF)FFL@MTx%=M4T8@8Cfo1feKxS?2^Yc0$Y3{tA0J><>if{Vh%nc|Y*CXo~ z8gH!Sd^&B=Y=Fno6v4PF(;W$d=6;*Cb(tGmM~K| z!c37ET-c}2tq22sd|(;^BgJzq>4oSJkjt`^u+dmxEz3|I*!qU`95l+#_`nr5^TJDP z3?GZ=)u*qs*ah4g6Ljw0ig1t7p{<{72jeUH} z8W$+43uYH4QV*0|HX#L}&w`^YVLF)re%Gc4PtfHsA=inEqypooK z2zuMangF-W9A+~|_2a`VZ`o%)_XQhw$yK&1m|-C;_1j9{k#MUb9-|*7{OZ3xVzWN_ zNut(!(?4V0BQAF;N%s+$^AWdJyQ}(!e(Do$*;l|wo-M5?*R7}A>3uD`JD0N8mue@?}O-h15Esa=|o{Zb~$8sMyZZZqi@NM{_4OUUSh5h{4IJ;w@&qdvR zTqK&C!pz7(7H#ayJ2Dox`2+w@zp-2!#kL>6Jg7~FcNpdoMuxP4lzZuXxz&*_)LMhx zf-%$~WBgw{!k;q^(13N`SX*EJuf?lZS8=1tS2u4h`EpiM<-)SiQAECo67mP+>_w2u zVtJ8j+Q=)-%KFjhZ;4NNj0!cxu*&r$Ao)p%}bttr#X-Ukh>*XH4cTV zy*pa&H?U&9LI=|va$mE>&godNn_ICB?N}d%s4gq9NL{rJm@vaeTy~|6965qH-rbxu zb7ceMvG+}ZL%=3oRPOXXMDA)HuK-=s*VKFEDpL=x<@2KJ-YOz6eK3}D)Jk4gsRm^s_y=LmA5>~^=7v0!Eudt418<$7z-yKrJ~YGdudlO@*PPYd~NaS<6dat zkC&&c`E`Am59M={&zq9&i|A1)@w}(!%0H6p!&sFQ?Z(dc`^jItNXEwlEeK!!)%2UT zvo+6)hb}oT9ekjzja8XrY>)lJwryNzIru#|(u@9=Pv9g&#iMKA^2fe8p?7Hugq{xF z3ylCNN2WV}@Xi-}VZI&A&vV9fEGB})Aq`{wDS2y4T)L9;Oh+DCQ@#5SaMG^9<0e}! z*72Jb&9_6lH(OLvUmP_>y7yHk1=Cz#J>1O?=?@rM${ROSz>XG&%wIJ%R)?lRS0t57 z1X}d&4GmQsk1{u21(}O@C;?8sO8ocM_JF&cK_Kq{;Hwb5JZRPlk1yPq@oYiw``5w;D~f z*|TSd9V{&TEEDG*CTNhMSTbVr zZ~#$MkWc)j)yM*MT)Dmtg{u|!oe0MsCubaR5}LJ4?5210p7igP<)yaqAbLVVokF=f zK;Ahqkx#!}MW7RHE=Ox{fcw0eNh|q%Az0k&FrntwG2KI3dZho;Q&qjmPwvdOHJRQ2~CBs9B z-6zYpJF12<;HCDR%f>*!{1OZWl|16a2UC21r59GxUx8U!#0#nJgwucJ>hcSNwYu7PwjM~dn5)2U~LN!A*W)r=k1}_>v~T_L<02}(qGLx zDDGwT=ZpodsM7AgaImv2kmt6!&p%^P*UYwpz)*{jVOSf;O67;qsKd$qEj200wys-k zGjV1lgsJ(czUydGUV59RIB|PNztek7C-!_PUF6Pf5?TvO~pIynnCLx7T9D z-()54ud9%~huEEkJozYJ?P=HE_5q0Fh1j<&UuZ=;H#&|z90yJ~g>?pWC)!?D$*%p= zxn+Kmbz!Vsxxk`zjNc#Vhi!CRGS~TqhYr25)u%Jp(0YqY&R~M@S{KmV`<#lGXdV6j zbuYd6;unCkBh;tB15FJXJ_MoX4?(JsOJ+X@J}Rz!0);Q~`)5}fN_zj4r85Zqly&W? z$yf$&ev3*;B#lT$Hxw5Z{x1`3m=Gk$MTlG!Zo6Zv3F-9XaJV(-^~(0|u;|qOHe%N0 zSlN!jFEuf}F>-T;I-(B%xS|uZ7gyiOlU4$pcs#m7xzUsXTm#6((<6F-%sV61wL^yv z*_#z`A-&`fgzY^i094i9W0O`#IPE^n*0+IUC)*3p|JQE0`6in-EYmLRyV+g^0SyG9 z778F9`!T=OQObkIAk=#M+qDxd9S>|QOtxkDeAf#B5Dm%h|Nj5k@@32H@++=H5`lvo zUw>QP-o3VB^*Vd$`De+*Y|EZ}`|?F&ZA5y4UH9@@TSb7E@$nJ1n|{+C`R0DwRVV50 zWxOXd>UVb(*!5F!;YweM8@zSuAp6Cpd@BIK#e03ltKL3zmFM;E;690VVQP%MT2N-A zK)S2(cz07ylF9jud+(SY(05H7Iu#MBNe5%C1d0CO>}0XEmaA`KJGB zPO=S8ibfC+WVwec?6NVLylM76d(Z#&@BiAEyMAM9_>P_7)Y(@z_rfuMoW*+s7Vp?t zL`}KOa@Q`i68yHuCnhqUz6ff^eA4PquN3GG-%bI__wKd&(XkKc@8;hKF_VoWqn?lWN;Xdz)f7G@3rD`akwJ$Q)UREPn%l%m}+t6pf4KlBtK7Bg8 zN<G_qcut zg0&Jvj?@02w-+v1vcvx#r&Ei}RY1EyUoV|Wr$rAqu3QFh9P;jV4iwaMf+rA49cxxNRY90gF26!vRAL0LB^+Shz&$ zeSO?`EV{tpQ2A#EcndfIlm#k01jFS~NzY2*z6$GuEej-3fvxJ%AnvHwJM=?9JIIMv zqb+5~c$@pT-`Mh%tL&U}rrGBwWZJ9Gt+qxGx;jP(q112e}BOcI~zT% z`bP^?4fERa(n~MdPagQGoi+9>`_h-c+&mk1J-knt%{l*Edu`~IHu9eB@9nTby~3IhLK8wK138FMo#os83Y^TmN=VgMR1GqQyKXen74*=@A_A`gJ`P>A z@JT6F5>9nm-&p&s--o@&otL6#e{BZ{9dWxLn2weZy6#CA-wVvA*yT$Olw1FlX#3|C zXW90GQv1p3J(kA4dwuVT_qSs7CfNAI>|r*GC~&=d;bh29MDV~#7IbEkxl<+4_hLX$ zZ6R~_B7ayTRQ0b^H=GlF< zD_U1cTF8Nj*JTx3xO15$<(y^beDZDpz%aJk5atnpQHRlxLQ85UIsm{qIXS^OTImHh zEMU7odh}0X;9_k+JEv;B5q6Qg2z1!D6dy7p5LC40& z+C>*!;6wP*i`G|JT}6?V#SgWmBQCU@wEnifxXixu{BrxnwNstQ`hjbv*rbfjcGJs5 zV1xr6ir9HS2wX|^P+79BAH=gghbnAyUa>>y5~~xc)~=sC(C$I(z8Nes9%QK0gwWkD z)z!y2I}N#>*)x^({{;DNmgAFXkYfL67f~Quw~wTU28I z#zk!=bw9{EEddUdp;k$Kuo2Pu9*BeQvZwyei^o_#oc5%r7E%z7RKSV{$V21+H4|(!;c2}jq^0~M!Et(jt|v{IV*i=SZ83vtsv$( zAoE?Em%v?DIE0O!3PP^|&w``M8v>vj!MpthUo{W44is{gOf~#6$5GcS zhP&M09^n1o^W9?ZNtBN}i-0#l=0*H=V1)ArmReeuA6VgM3Tm?lnh?2=IY>Pmi^{vO zsJxSj6sJihuAB*^F>Du^3sHlGomFl5zMa;qZ?>I#^Vc0m9u)%mGE{n54^h_!wfU1kTiuCd^1I92N37#eLgiT$m7@FjMzPmVR>Yko&n zwT&VK>ceaI+o;~_aKKnZ-jHu!pPX%8c-NLl zCw=NTVX}$`32(uo{?XaPZ9H4ly!J~SptNQ;@CnzciP0gBu<(owm50}#$slU|-pCmD zBWC#@uNr4FhGplkqsE+PIsj)GL{=;`tKs2oyz?`1J z@4JCtjrioKQBSeMCKwlXnrg1Eb=I8GJoKwCHN9Dt&Zt`v zV?S|w(uL4xff#RExyzo)Ew&u=%KJgy(qU^`*HCBGUT(jcHN?jH4bhx`b23)HsEsR7 zZ+z=J-?j1IeApJmW1XBc&4OzxT+wy7sjb1yXdOW{Tr@R+DEo3O4@5N>=Qdw_eUlAH zhNB;q=?vG#^iQ^zZ=G(xnzzcnwMqN{nLSXbvLh?0x3c7J-q0hq(ou#=JN}n2x)r`*VwaHjcr2as+{Z=nkThONn_!)<5L%mwbvQX7fWhw zSVRyTn_HF`_`c*XV=1@7mNA4~79~x$Pv%1NmF3|2?qt62=r>L4i#efo>$lv^_Rbmz z=bmAQ(6bli?P7oXA+q!twF>!b-m1NR93@!61gLj=)w2lY+9xx|_JG(oEP5N)vdi#1 zNARFn7*xKI_qkKvu#R>>7rh5!mL;!im!_=iA3BpEL?`7-Vo*q~=Xl3kSekiGAme+> znT0Stowd9)bx-ldT=EMguVjofw?;)rzxOK47w>Bld-?c3|6opM5c&tx&mAo#U^XEE z=gyxG)jm$wU1o?p#*Y5C8gSW>?=~1xfuZlHWCO;Z-;+QIcmnbZr(&|G60L{>~AuM0bAi&{je zeQ(ZKCrxNBrO>zehsy0&JBzJ11W_T#Q<4phox-MSYg{V4#aR2sKe?jZA@?97Aws|kC^hlJ@)J~Qp8puI@<1>KGa_P z=lfO<@fpE96+hZ)B6DrW#QD0@MYsB1I}fO_TPGq!)qah$yQll7D$)O58FGjt%K_a>WRK9Mgj?8~85iKZzyW^67SdRrudR0E(ak?nAVZ(Cz_D@`I z<33w;*l!*fxP0l7C9C`y>iZ7`nXCL`EAOwhUB zxTuWom1zBkkFlJwldLxsytbW5Wao2fi8x!LR1d10bjJ{o1@UNORgFc1ES)Fllg{h- z55=X2!=>+tPO^R%-exb&eF-joiFC}mE_ZElrDekrYlX^*3|4veLp-u6Vwpz- z=>tE-ZtuP|-yV5#iH)B;!}dFCVh{sJT)x6}1W}%n%5DzYH9&F%lMZr*y|T|dv&;$z zj~aq;nvBepK6X~NZNhNw+WrZ)48yTLUFGrO-2({uWn>%Nu4M>Ue6EnWS_JH2TJjCsXt1Oki zQ~Y*4MIG&2*2i`{2ws}yLhoT<4}bX z{Vj~_JY=rC+nb8VmeHlZwZ6`z54Q)TD-V{Q9E7gexz{3IUye`XZ!vKqWQ;-eG)`@D zz~~8taqRD? zdjSfkyYmAN6LU;6PuNj7stfA<_1>Dnk{B!eUEIselWkC=VoOSTN` z6%BWjiuyt#b$Ly5dbT!tMSAgV7_p7T;Ojkf=J&t2oLvHlhT({1&?dw(sCT?QdgC~QG-is@d$=6y2qWV(WEBU38Jj}&uQti#Njkpq4I>Lv#pRNA!S`!8#(qElSUg{$6 zeybm<;z(8`mhNm)4l47*M0pxl%~cWQYP~$rWx;t5geXt!)6a+4+0U+Y)Ev;drVk{F z7h;b<+A$cb_zOPqS)2ap?>J)qVD1LN617msiH`foC1xv;nh~$7@93-7_wVMG9z(479VSsC zlSs^(EZ7CMG+A0rUL>dL@)9dA*kc1GPPb{-+(Mk?@oZy(>w=R!!f~^u2+P^@LBsJj zmE*R^_$RGH8oJiuC9URgW=S~fTdVuX6Yq_Vor3|u+X?LP+=e}tjHR|5j&vTh9$mKT z7L0bRC?wp5#s=6+&;HA%O*_X14H{^%h?LJlWc-)+_gFfS!t&7N{&vnN%YmEqIcD+I zMxsgqyf7)W8xj?2^Elr=w2#xH>pj&; zc_+t4+Su$ko0y&It~EZ*%jBmowfeyKSVLTesC_-$_nosw&_7{LS1V+!BmzS7K!n3R zudlQHfVc?eKq_tDiUoTCQi2~$&9aBCo@hgf_NL@*anpBC5)JKpi+9>cjQ`XJp7A53 zkds2hkWayGYz(UR+4S=-g4^tvvlr=;RmU>okw5a*YTFeOWeHSSHW4lM1NBl{MWmIM zmf0m4(KcZCNK{EDAE`{`RI?bmWb2wb?j)19Ry9^a?)ok3c13Gb^tVRFO#9j;X%jcE z=dLp-&->Qr#SXs*s4f&|ArnbmL_%t9-sZLSp%GERtOwae3JxjxPre^L>@U6OY0g+} zP}212W`_BtXrBUOQ*Wk_CU;w@(z47K~QgX_3w#^bcVero)lLi3hN^ z*~x4tPn`HA7S7WEoiR)z)+Y|CfkFPG#Tsf63z17_R$WnQWd*yf@6fYs>Q$d`F_q)k zf~I$zcwORR5Rj<&jKL$E+ej^ZJr^L**<0W(6{bj_vWC{IJ((4)@5Gtz0HHs>X|E-* zU%FKFZ>g}(+yh`qr(KJlI|T85THbuyu=}vho_#Ua;aK}3Qm!j0vwy71w@(l6?F_fX zZL3jgzxqOeuY@RBKmOS-Z1s>C(pcJ+!~56|5n-Q;zP$|hrz;RK7a{5`reS?R$SJf< zZ4r@z&D~KF4H>)1l0AzkXiu!)Z_~&l+b`OdD#Iab%I=>OYqt;UV>RqSpNj#RjLR;` zOyFI%7^$xWj7d5RfeNv!Uwr_CKzqM;-J_%bwE948^}QN~@`11BW0fLjc z%36yvneQ*yW)s-gE?29Y4^`QCtjqs#^*Fn6>JW><4}sdCBraPxP!@Oq06+jqL_t&% zg{9l~+sB@L-?A|Jll72OBX}9}jwW^~JvOwy#Gd}`19sbOw-E$Ko;ZEu;4Y5&nOcH*)Jg+|6cz5bOy=M(%T zn+NJb9U4mS+I`;x{-`4$Iq4R36hh?8wS4CEpSkx0*SqN>ybuROBb6)R{(H-oE!!~P zhs3q)eI0(9bi^5i{s9j}PEJnPiWMtr*RNmS2v5lE{U_`VAP>21r$GI{epO9r7!7u&85p^ZFSdnB?$s)uqc-uq!c z@cPRy+UU_^@b5m-hYT?k`}zC1_Ta2xKJJ`;@GPqt1J@_-y!pD__4*zgF?O77DX*}< ztlVRnSb|;8KJ{BB<=AB)+97P;sz#i=ys*Odb1s2<9gbJD7!aDo%kqFGp``{>y-ROf z1D*KT#{G6ZR_H>=p4GM>inysbMCBKb$gp9N!S?Ufd6rC>7vg9|h}El`+TDae+Zs2$ z*iKTu3IwC_kDUSft$Fd^mD}yby$3Ci`W<9n`2%MUum`W0U_<;|x$5GpA8n=W1rR(} z{bR8m5J5%W(w!d*WZ=c^P++iSG*sK(1uxmyv14q^m@%%l-dyw$jM}XvM5IG$(Zcua zx-Z{rkyGI8A!sWR?|a|57moQO*@oss8J^d!9hg8^(m^MU(3LgK#J~*S2Y~WU4#6=s z?c`4?gBa);EO)b}`y9p7W%7$02saRUu(v0nxF&8s7c;KYke9Cv7Xu z#Ytg6s<0j7?aUd;+BX=kly3XS(VC!f=z0Q`H#Qd1AOEH29%fJ4Dkhx3o-AO4K;}E> z)3`*s1@>uv0u>ue(Xqx)tLWd7k?#c%AUcO`o3E73ZLTppx+KoX1f zW(&W?f|}h_Wh82q?Nm#EzuBv;Ug|p`^qmI|*$bQVnmF`RGWlqm_M|IxW+-CS{)d;^ z+lx2iUww}A!5+h2`YQ+ET9iHEVHl~2;1N((-w0vIlh^~ld(>9;A4d?4NL2pymV+Mr z$#whfIiw5I(|g(I{wX#kC(ULLNwZ6{5-bx${w5A#_7zk(hbbAT6o>`s>;z<W{psvBh?GQu}y07pdq`S{o9}47&L6;`_&a?xil#W z;x~!Bwf#$6t>$*?9seQsK^w(O0*V?u5SIe9rJh0 z#>S!^_Zz)I-8D>}FyVY=!!O~=ht*-2!XXbt zQZcrjSagb?Eb)5z{_Qqo=2eK+&$A?~T;;V*#%x0760gO?Cs``dn50Kj1Tcr>p~{)N z#{uuQ3iU5o$C0%4UVG2fHsvLtujJ*c^QlM@zY>Vp4x|sWp@S~4Wpkgizy9rScK6+P zyW&Rs4aQtO{2lKBz}2f(+T%L|ZTR`AR!~`qemuaog4o6rk9!$Nd&D2!vL`PZW*?tA z)H!XD+1Y`2^< z%o3xV^|K>9JgJ7hljXSxqvuy_xBD08+L)wB+lYQ%PJEh=v3HHP58{>ASxQ`jEleG4 zU%0)uC4kIbUX(4QvUkrO+rE9fMM6{+78Tm|ZQJdI7hkfMUV71{jvr;kgU4GgNHrd= zQ^>llq#+hzC6S3I+IMYi>!Ws_`KL`gZ+-ZWz44W2Xsv(2gy>%JY;!BL18$c=zPqoj za2g-_MIz@0Tsj95(d`^=Ew8Dr4hRehd8@vrW(Dt@P1`@l7^SNZIfNWF0H<@8cvDYe z3#Kld4+&yt52x2S7Sf3QL5+KJpQ9LkqIUX3ANq9~rqg*4 zPQ&gxu`0>2|G)nAFHNQfka~1gdHDmZp3iHMF_}c|ssCFissva8ZsN+dKeQxoo23pu z%f`-{<2Z8pVwa9w!BZNUe)0h=L@r%gA*Y)Bzsa$Co^HR z@WT+lzO!MM-TvGX8^ksy`7>|5;T^1C8?QljTP4PE{c=}Z>cYot{_@SX@V)nJGVViV z;VY!!^@-}IYY|1?`u>mXfl30@^hvdyl~n{uk$~7|bTxoZCf zk{)O7<^4gOJp$tm3>YJFAP28` zftEk^Cfk^Omc=27LEhwh*v4O3mF@?IM;Gk1H*cF|aajYbqMSY3=;PJzCzABOR4MFu zv|`Nu{WR){Wbxs>%fwRsHGU4J)Yr0Yu)q%N+HR!>3t1EFlA-@u4-K3HYpSYBm%s7S zx88pI;k>~!XU3EtI#dBR)Ts3$kLhG}8S5xf>xVg`q1I?CQ zq+ost&>C*PLA_V-Oxs}a0NW^TT#Z-V3xn$F-e0(C)n2ZZ1P0Z0Cpum9?FX71rSgH6 z_|Yg#fmA}Z6^-UL;yd^fT&g|?u)0taipitla9^@_^v+_Pw3Go@Lw80KWC0HtvvMnImnKPzaOiYX`R(<0oU2odB-ah`+ zd+g6W%L>@b+nfuPGZh)PDGFMDytzHM`=GtFDbFUR#8_XVvnk11UaEy8_O;9N+|`6F zy(~M)a&SAlyR^!-VNh0#bCo^3dp08J-)`7%(|gBSCf?+j6mG1PkE*5mbf&+R&&ejQ zUsNv{@F_VYPusm`x+fKZ`lt+j`InwvY#VU-nu7X44sw*V%}q81y`f^XD_}b!_yc&8 zJB0K?4t74np8DTkIl%_@O%Z{Mq{&BQJ^l34cF#Tc*sSv}wA9{g$I3{z>^?~rjf=m1 zqpr2w0cTkxW9uvC$Qb%cJ^rs2S{X2TP<5$YIerik$a+D%`P`}=|91O#Xo|CNKb=MH zEw+L=<;IV>=K7o8+65C&2-XN}YicBsB3P5=pH?UO_b`k^_r3h;OY1VTb3$utYXVtU zLVTjh*6`W_2tElv`Y{jWfKgv+Saj%G#|M_EldShd|7qzU5JV>77Yowf0@m1D)cpz0 z&+vSk05QMAcrBD`iQmf+wXeQp*|LRC<>z}Y1=R-S*DaFf`kB;)Jun^dy0QbA*G!u> zEgo^z18iozU8@8RL|nO^x6$vKJ3cRORT%M96z;Y`)8^QSsndzTl)*EA34v8)v5N7E z^Y5%ZuQZL_RlY&6B-i8^{)t@NJik1G@xYn-5#Y>NxG4It@>c%b_eQNpg9 zGKA=-+TSb0ME~<{3_9}3=-PGMR#IUrcOJ0UH|E(-w-?$1{HE)r8PBe;^>F#JT=&73 zBja}Dip)NQr1DiK&D~?A1&-kKSMZN-ud{z|EwbT+KK0z#rohI=fuIk;iLWhXuQY@V z9b#V|nri0{PP4P{7w-{5%FE@)`wCfk!woms)ZuAXTe#W945*GswwhG*<%1?!acY)@ zGSB_IW`}x32pHY_^l2m8q->rtb4aA~0ok=Is1Bz7+A{WxO$>Yb*2ATf`PjA`&h-ji(g%es&A{c`( zsKlQ^(e|~Lm^sX*UhxgF8_GG-+K>XYzECcz=UwR6_iA+%V16M`fyv(!7PV9f6y>o5%8B%M$4Wkt z96!8wz3~eh_zkV_jV&ldKm80`hk)cdencp^Ec%=H8}-3l}O3pOTQg%5Y`sMt4eQ z5PH{)R={*$h|^kaOH1n2>!(b$FA0$Q!|n+OwAt58zDDhV#tB3~dBF}l>(U!-=!COf zNYe&biW)?~qNcU|L;)H^O*5Gtl2;SFh1oUD>H1s*w`J9h;?ntX4_Do}d8&bT4~klHMX)c!Op@8ezS9}>f>TsTGd+ZGWaUS4j_W%Kc1i? z4a_ka_sZx6cGtdwFH8=db!Kf`j2$! zl4R!L=e@-W;KH?(5_=Q29xkDFrbyDi1^eIf?L?M0fPeLd-rK1WNU0e-kz*~*IR!7 zK*BEjr>Ul@5IfCfOMXB&BN)MYK0rQCay<{sI)!KF@;h5dSNQ_s)H||{Z?*e)UlIVl zr=<5ZIk@elL^`|4qP*a!mGXGHlW(^2-ge5}#rd$1I4_1$w`hD*NSA{nd}eN7U>NtS zm~$KP#<}U;ci%n8`${THha(ttqdTTnJGMJE=EJD7xN@x5fnsLp%FlrU{Jpba$H0dq zzl^fUX)j@@8yF;yZp_!xv#+`_JUx4G2;Q?A(US*APaYZ`={RylT5>>1z>Z0+iGHe* zmX%gd6525ZHPK8znbK5G`7tj9J zJJ$xW58X55Qa)ZEmZIMH*?-=#@2xMhQOQw6W#b#33)MD(7k$tysqT_8s9&-%n0vpH zF~=dyLo)|q6+YY-=tN_rfBuGwBUPE=5KraUHLGTqUV8ndcHi@_h4$cz-B347ak%kJ z`t5UrQ*GwZzBU*YLM+5l^Q3S09l4G7tnTlA|dh9o_D&h%NMYb(lw2rHqm&F&@sGz|*#If%6&?p@9kYsn{y zc|M$XrqEGic&`2IF7Gz~Xl?L<>fx8*Xc%R7Q^5UwJlMzYJ>()5wh94bLQO99?-t>AU`q> zz}wsXe0Z<84Mjy`@@WDc4bhIzS(6ANl2~Sl7K^tmnRm_i9!pH?pM6$!Wkq9Eg$POX zTf{+@o7M>F$wB(EY}d@~(Be40%n#*`7H2R>JsTIS`2;EnMxX8j)2FsNvecdMQ)is$7kuQu8|b?wp=~z4|_dYu5)~-)ch! zCD^4I33d@~YR6=zS`xn9nM5kQuCZUb}e?0Fj=6NS{s~^>Q*I40Bq&peaA96<}Hm~Pfp%k1kTV= zA(9g2(B3uq%U|D@GJ5)m>gw{oAXN=wEr_!LWGYr&+uiClKLBV7O)BHLI+)_6BTV zMZpvt#yiSJ3rQ5Sqn3rlxl_k>w4%nKpxlUvi0!CEMW=ho_YGC{Zglta8H9ckBk1SK z>x9hxAx<9wFiq9A8zz>hNinqTCEt5cl0hM1MH?2p^Sj%A@|*C~^!|5MmX|uDtte<| zSw8W)ru>Ix@CM#tp|sbWHlF*DW&~o_p;&+~+;_+%qdN$jC#vUEC9P)gF-yA9&do{! zk+>~6jyv|sWW1VwYtR1Iub5_$(W{7MzS%}4ggZog2&^D0cV*RD66Gc_5_9-?7>f@= zw61bnN?wfu(Gd%Bdv$H;-HzPr?Drih<8*k3`5B1)tyF!3-`sJq*Nm+A!p-l_pAY;? zgXxtqN*Ww+D9E;kqY1G)Zdu(7(g*oQ`TF0=xxb8W777uwP6)Z^T*@9g5q;d4{8+5H}Ic?O}M;4o@C8i-s}q)nPI;WJE}UjVcsLFT$A32jIEw9eBju{G_@ z+y(DEe!*>D2n3lwBwe}YaN5AZfwDZ6CA%!zHTymc1ke?;UMom&J`@f;H3kk2n*iiu zr<^R-K$NRsc!^PXhEoz!l5K;M5RUqE<;s=TFC)VelZa5py)T?U+PPu931c@Id&MJT zmA+h@dTXS-*H1ccLPtCli|*}g=bG{7MfR&1#dgcN!<^MS0ERN9(sNblR)3syfg1P5 zD*wqFA$*$Y#ihnjU+Nu4BnfRo#3&5xe^&3d&z&>a`HWZpi$fMNuc)Z7`|tmOy}Nj+ zjl1jDwhqI$X!fOd6*AXY$Wz@m7|qOHkrvx44oLykqF#E|nM5;uYqB$$obKN`gsz?n zNXNyT!uK8e=Rf}$bnUg*O60u{W+t8WFc@T-1{`#Vox%4ZR*z`Sf8X^a5b zO8i|fB+P_4Y8YHe0&^N5l`7i`-#dEbx?IG9`<5qqlLw-uc@f=weSI0(9|)|g+Y=fW zmwya5u6JBMoO^8Q2=eK%jX%ML1x}wnJ&;LKGiAz@1jJDf0hF#4fJUt7yuG!TAG1j$ zDTeGp!4Loiod3k!T|jisx*_8xe>WsFBr+;Ku?|vv z%G4_8io^=E=qNd25i%EI?TTDm&qMhu(pMxN+tL8&&6ALhmtK4AHTIxywBP>jcNPW# zMaR3qBXOt)KKAqq3&OYh0FYNHV;w9w!iVlSrvBc~c%GKQtq;Q3Es8zsUw&(gEyf@E z_vVasF~?P|itmm{UA-oYeI+4r^@NbP_MH+)ZGZ7jz_!-K=BE9=l#*jBUuQA0G?4T~2q-ots%*<;7{sSOAi0ymw;gKE}R zB}h92#LGy#rGf<;R6II)kS@7mnAX@RgEgmn5vMK0%}_OUE5xC5L8rp>67MP;pCcA@ z%l}Oc4jLL zKZfC3I!N8wL@{HeGiocj2?Zc=6PWMj0}h{xi0@te#>+1>-u|_3U69l_eLU9djqxdI z5*ay*c4g?O)V6Xy#m`jnX1MR9OA`VW0_&fE6Bf=j{FXT5RIM`ATT%eAWVdJGK`-8? zKlgyS>H{@zl{nML&p%*~J^r{QCnwwge*4=N6&2;U$V$e&WW$~AldG?p}`S#DN$J!O+bDRSj@3HrJcR14;C*@FGJtVGc z^{3h+=BkQi@uL@4;EgQMeoP=2ITezXzxv*jGClVA6ZXw--ec20b(gKk7;WJYuMYGo zvo1;+2E2v1F-weN+o!`rU6Z+gh`T1U(>E6lcRg4+D4?;SudAlR=Wf~RnU2NgtN#fB$_sfH@Y7_^3=9$cy5n}ejvYD}y$KMVCJ9*FB}pyn6&(Tj!`@;zX8~Kt`L2fJuCA)G+eywv za)&5$GiS^q@W->x_2^x9-DRyQro@rWYtRmYXnHrW;LEG)5G|FhISMabEkh)$>E)4kJDf9L<_nP=^@ zpZT<%d)=pPZT4i|fWwC$16aHFyW)>>Qp6*9s*_*+v#`1k!iJ)Ooe0`8*O{cLqc(j+ z65DARp}=6Dv;8BD_9m}7Q{FJAR5mt#8>Bpo?YC_j0|N&oZr_Ws+8Q9^GBVmsbhASs zeIep-khvt8<;({~-z^PoY%Hk{4lZLomM&kq6pu_TrsjisEP`+GgPwTv42RyzsLBs} zK|w))kU3#aGZ7U%0ardFU}8N90>B2<{_6Ig{3w(e7JpYOpHxB=Np!g z;M>-wTp4dC8Xl6CV2|8#cQ#M2j$K9McD6nr!1plFVC;YfC*n$;-!x3>3WqqcCAx-x@vH5kU-;#0&nb(5Ue>HEFtt=?D z_ivtty+Rc88A8o*!aib9?Rs2TpY0@b-c-E9SWUk^0vE98&0BfjefI@$G~yBSeh~S$ zgvk9)v?M-1&OFAAqkqjQAK;z9iiQdPCNLA?_AcxyZk#w_!k@YS2jK5^y*GRI>@d_~ z^)OaqqLj3fbeu^o^uSSr)WY+R+%5l>G#@gbFyTQa+nwr!+S=M`CS7~Fa=ogFhXfW8 zdwunl3b4~_%Is8BJ`UGh^U>Ss@Eycr~f+}sWfJ%{k9<@Ul4 z?r$n`=Je_Iz)yc_a`O*i>ylk64@$PlMv-(i& z-$uZj`{u2&>Fh;+@}_g?R~7&WUaAN;Yy3d`%{Sk)nKNhF*bA<-qRH1-by$@3hCY^o zoTy`cha8s0!}d((g`$7RMX=`H$s~Oy)yOV<$d4irPOlgrk#RD49s7;)*4+IlCAN~e z<`ztMpdbWfUJGY_6XNx6d9JM)!j8a4mk;mq-_dQ*%RcK$)>a>myCDufPu_o~#GlNa zJ9oD}nC`b@avz`h(`pZ#LFoRjVS)w{7$QKN`P6aa2Gj=z{u!=(iYQN#MCa{IR=3WX zkkXZ71Jw9efZp?vqIEbmNJ6(nz|`tBXMgHbtA73EFIT^r*O-Ki{VU}qNba?N(vNIf z9jOoDfmWQ-$pTg1*S^1k>q2^A(C+<{h~UKyxqcNX-xc{cFH6_bnAeFLuP%cHI2!uu zm)a)Id-DNoYB0Nr0kI@ao{hJ&1dCetU#kmGB_8(<8$V%!J@d>ncEuG}IQ_VTJ=9&u zOi}>3G1c}hk3dk_0(s)Zy>ir6uP@#6ZvKa6yY{Pnxy7Y+*SSM%R6pms+x4jrnbQaE zJ+q}tm)Upj{+eC>$UKn0a3c9Nf7i>~p|_qisLdPM?3$_T#*tkq&LPYCKmJjw z#wv`1Q?WGvxz+=(s5?#wmjEZ#dtMKsPInRDYROM~%VT-|MJU4?(9QoE&iw&6bf-H% zgV>u5P79YOGzcwx-y?2mZ#f9v8NM-rt^iQ~3CKMcyHvB<~RXVq`E#+H=2f01O#RgNO z%dfYCfThChN!A~KR|$(Awak8ltWSEnU3uk|_M;#E&_4UQ&sluD&o;>Oa&`c(cgu;Q z)(ba~)f&W@l$NzdL}jNfsZ^wpG`28x$D-+_psr<^#@F-d$n+xs1m!&<#T(< z8NTTyIKG$p_9-q)_)d-(6zyPrS#|Xaxc{1e;yGys?hu!;N3(ce?QOVd7r=v5Rf(r( zQv1b0>o`s^cA`}#GilC}Fi@JE5M^`m7$?i{Q;}3}r#C^Yt8G(C#{%;rknuCbC{KhF zETes)KrRW?_YR(H_2E&L@Xq77sEx2$7hG)Ly!T#P@b0_z{U7|m#-LmG_EKlE4KEI4 z{^F{gcKaI}Y&h}2uO8gn#$}~gO0Q^Kp+-k67GQVIif0WwqQQTxT$uB{w9HYXV*d%ZXI-uoRm`y6T!NCms3d&#{H z03gCElC&a!aqeZ!)uSCK`y=tn zM;W09Fd>5HE?-_P@ez`0kY(E&m*5~*9|VTXKuJ$!UYwKbr8qJ#Cqag3ZqI|vO33xny{pE=6vBJQxHg@3oY z+~s!ug%{gee2$MDJJz0f;t9Lzs;daa8S4PMwxHcKb%g^d%3Pf1<~4Ey!i~~O;0R*au;>sg^HVGU7oId) zIO3C{)J&_M`DsgAIoC?o{?)Q_W?H|j0e0)Hx7zFT=Gp&4bpkQyvX&v3JgEJ@3&+?s zlLp$nHMw^4yIbuCuWokwU@NP$J3cnVetz}j9*GDvzgau3Z9?)onoD#J3W2k;VblJD zwsrp@Td?o2Jy(d2M53Unzr{(XV2z%d8pZC10Ozu|h(49h>qL`2(Vl-f6`GQkVzE&X zj=1y2{Y3Ng-gh&D%ul9HULM`-+<^c4pxTu^>(Um`7z3qiV@5Na#H-hIHKG9!?fvNj=_jL@69wf?3PJLC?9G-m~Z;lUR z4gX09_8WlVySOJ2oW9fXkxuO)t{&Gp64NwoNt-a?PG*2C*|mRCkVPW)w${CyY`ibS zH*izqlT5tlaR{S80V#Z60Xw*8$>POZ{2kXc(Uz0<_<(@g*9#l_Hq_Kj*H$%U)>dKu zJU-NuS3eoBbxmziFwklJQhHf3{?7A2s3}1Hu8;sU??`+uPIMKdr)f<98`6Fclla5_EqhVlZUtYhTj63#Ll4r9!SZW_=4%^yrxM{zF^o2F7Y_f<&u zo~G=t*VNQpOYQz2wkqo;OqlQuMDP#!i`KLuyN_=AC?Ry`^$m)yMq--ihzLbUyIE2r zP_sDa<2M-}Jud`_n1>FTYb(V=aOSI-SaD1?iHLTC^p-4Nyx5Dgs}*tFoPV2;?a+MSi6EG9w)0rfoT8lAMQe3oMiV@ipoA+5T`@4@usGTX{$hTVE z^#r@|5&cK*OZ2ZK<10~Bk{N1ng$OiYYO}UY$@N(@4gteLa3VYUJPS(eXTd99w2Isn z79N^pDG7;o&pr3J)m0T(+}X2WiKF#N)9y2xVY20)!K{VvuePEU)@s zNZihZ4U7)6tTd;Fb#tII2wnNAZsN`z@!83Ba58QCK)+G@Fl`TJQLsU0+vP)&OF^E) z*}wDuM3M8jGq}Scb;d&>RqsR6`5@chehH!`-k}XZ!r4n=cgj#e7j!O-`aOyFhjJ`m zw0QBasok$JV4lf#y4x|BOT;|>*T8ncUe)|$rw^ZY@6?$==}s;8Vwcx2DYIFa|A5jP z4_B`B$?|uwysh?#YIJ>tnCEq<#y_F&IRtvg>qvbq2pv4`^Ke8aTK67aaCzc?A&{NS zZvPuDl4gZ~{=qXHAtKtU(+BoUNL(GIqrEVq?Nv&7Y5Pz_c#usXe)ba^_PRV}l%@{m z)am4{=pb@yKy-dALigiytB?+zqzNFdP{EuPn^6d%7g7WHg0G~mMYVN?Wh0I6V>aws zW=lUizj~{kH*=Pa8a>)YYD-B?b$RKJXX)*gD*{5nXjCueQPX?f{X*^Y+LM3afW5FG z-yS{~WX0HqV8-rS1t4)D^fBmJ#EB!jwf!J@MZt@tFFYi!MdPK;&H=Yjkpx_e{Xz;p z{N>HA)0C*ca#{mg+&yO$yjuVS@NddM3t2eqZJ3jPGRg4S;P@|evECXtvw$#>0Z2cEg`+Z zt8YvhH*O@A>BsdNj?PXWvgp34$0PLS`f`x@#mvw@GjWpqB6i`Ywvr}?bUxZAUd9A_ z0$tBMz6k@Vr*M4$5j)-r7klw`wE(oX zyXth^=|CyT21$=j@Du5;rEl%Jl_-5~00aX2#4D?`W1+<=OYiMV0hUe$Vn8s(R4f-{ zkgNUCV#S2*;qggoPh%vE3ZWdB{#>9C1hEFR}-m zT>Ak0AxJRBV)?J*+iXWGsi?ACjNS5zN^MDQp}o%bvn2=6m#e!mbQ{W{g?E7d_Xs&H zoURyu<>N-1yeIWfVS0sH726X#yIqj#x+GD)!*$*b|NS0#lE3vJ#Y69#9{v*TJ1;mW z_=@W4Dve@ik-67h0)!a6*Ehgm%>gD~0N4G4$jHbausv-L&-SpLc*DGT^P;M&tNQ?i z(}dhnb@pXZU&_1dc=vtY6(Uz^5&rrfeKh{|XoNl;??ULy9cPSX=Sf4yy~8MfsX?RQpc!hqgCK>@M!_@+F|VqbbO`FB;PNGKR0W&NR2 zxukSe6tZoeJcKR{?hYAeiE3dB+pyShyL4c(4a72iaB461%|}}@25-@LoD<>EJZbMW z9E_6!K=?@~)BJJIzh*4euR1iY5$MGuupvn0upG0GO&nxjE3L4d2M*b)+yZ-XOTIm` z8_B4!E;bZ{=x;^dtKyKFLnCxd7oSic2J`Vj5o#hF`oY}swaB~9z?^Hpe`aZD73n=F?ZAUod*WS+B5!ksx!MXsVjq|lV|>f!L8p{Sypxx$oji*;Ln24 z)#Ifcn{>dr$MQicMo(F+y%AMqWqn}OmY_!$Q>Ax%{?O3K)>4l}=rVqTa;p_mXM#Ki zVs`p+08b4gE}eI4NfWk#$>neh$oyFl_}|gt$n|R~*JYgFWwPy%B?qZps?hH)5ksr` zkl!lmliGG)q?1nf6*_;vz_ejCr&FZvM}8h>ZBbn1x+-R&>emNe;Ms>;j%6C4sU z0dbBecWqT%#3I#%l@3%!8wa_3m`V077+W6S68OD4G&FS4f&~jiF)C@DyKmgDmi`dW zmk`3fcj_C)khO3Kbb*6cEq!rX-cv)nN{{X}upW{2;PhmCyT{l@wl$@L;L8B!UBTlD zsklHah4rn9+8t#IEYa_%sHjXh^{mXy%-oN%2e}@F&=JoB z!jW z{Qnkgv^chQ9d3|=(Dng%+xat+s-F{oAzf=f0)$Q2Qo7eEfuB$aCNXpT7 z<$m_KKfu0#5Qy2Dn!3Ul9{uI{LwZN{1+gj`1wyZ`>(5p?P6ar`PMLzcA?Vld0K?ym zHTy4u!@_?1?z`_E#F>mupFTaDXo$5Sa2e@s_Cj8_lJ7_2c8@{m8cYznCO`ue-A$bT z2y&L=myTFu`dku%U_df|1;GCYvR-amdxO-ALhI_5c+?^JddU2QCdl)7jggI3+gGH8 zU|e?u2wG`kqB@Jry^a(2Cb9T<#O}6yn;irg%P_5y`<(?F@-OB@GFh>9uvmn=`u*r! z!vI=J8jBQ=hpeu3ErLS+*tHK@4oZ}l+N{0!!tXMHDB-#kWV4AF>6;F6L#&&0w;lmN z9tMkJ`*&g5gT!#*)4F3{`|4Ni%U}75yH@*;t&@Au4gSzt&10Wh=GiaV~DaM0$h+F_qsl55Kfsw^uZ!V>sQe*Yu%9pC%2cX_$| z_#^#$5(HNPTjf4JCd(2L^VhOFp)2}VWome9Xo{L}Z23(U={Xl_vaNYWbSBgWIMqqZ}u^Reh*o{0`@tUk0UrX2Ze<#Td;6pB^@S2 zuC?rgP+kwVbN(JMK;MPcRrCr72}^=O(;e-tZ0F0<@3DZH#Ezb|yE;OR>rB#?qJGJ} z?A8HnVcVEzL*UHHL5eMFr8{1d21~fr@TXo}?9aaJvAEc}(w?85X%|j8 zi^yN$F8{7THWc%rB${bu5Hkl+Q^eu=HWGqJM39pIQr=mIklav0E)L zn|l1Nvb(Zu((xy zInU~u7;@h#9qT)M@B~L822SjyvfAK=h6M{(tk}r4lJJ9I^hqa#1<_2(gnFt50PD7J zK@?DpU+mu5Wg`F~gdPD9o;9?eb23xGZ|O~C9MzfR21@`4!>9x7k#>onLLN#2O-XE- z6Lq4&NQb*C0hnnUl^o2pgb}b&UGMHb=8v^Mq|@Ash53W_5{0dbf!Hrf1N$B zhB0LO;4lb>!@MJHuD7_wou@-wZ6KJ-=SF5&I^W74mfeV zEZ3g|GQSAM*89om%L|w^3l}b5?qlnG6YHcW#&^l|J`Q%mMudd~hiI`-e)Gx7)9#a0 zaqUD!f6z6JAVX~*84kA9zyzB`%kQrjcK z@StR6*q0v8x6p&TY)tijOOKQ$8yK#{6R z_36n;2t62rY0=#0?H@n>u8ll*x)Zg3K*(Go0Ne%m)Lu=Lw>q2${;t<4T=fJ-yP*J% zPvGXiP_xcw8vJD8(xoqZh`pKdoYTI!4|@2Y?m9&S8LVK5TG+2Rp27bwV)HaOZ3(nn zMCAT)6b0uqd}B{J-#v_}$2z7GHAPv*qrV~z$iyt2P$ z5-vm+)mA&{fwJnZeeb#<0hzPJoa&&?Hbjdv0?YBY*6g%TzqH)?VOUn7Me9oH_#f3n z9>_A_)`SqY7uVQ#hlSgh=1j1}l-@o>e3^51+GszSl4YNqHq=sML2@JkA%P_ajJTHGxVt9>+L6PS z8v6Q&&o8s@q4%Dg6zTl2YZ1ymLWYU;h|I-SRbfCPZm%MLmqqPdZvm_3TWS8Drg82+ zV5t4yckZ#-vu7K|POf|8L%`$S50;hNllTz1lWjwoPc@Bs*GUiBv6RK$Sy?0cdKEO++=bFeFyHHf^#8uAgR+$^B6og|H3r z1Hpw(p3mBlYI6bSt(+Yd&KSz0Tr^!D$Gt>elms=A--(>V#h7t8DUWREI9}n|gNWi^ z$ovWn*#5?VhJzH^)0a27Z-rBOkxBAb zxbZT$D_OCZ1*1Dh#IA{?Bxu_ugHJg=gub_SYIu0?f|?$H(8b{*NyRs}pPF`EcJhD3 z?S%k@%h^Bt&8Oe7*9uCk58R)0+9#4EdKEI?Ah)9|^1sg-Y?n_MjL6rANZSJNGI(3G zInVBRWsNPz;(bJ1xHH7^#JVljLyJec_jJ6Oz%bgT;!3+HGv4kye}qlWNpqhF=>>oZ zmm3=xYI@_N7_cXXB=@mVo$pWgcxyq42BGL#7qb}^QiKJ z7G1d8qV{dD=!0AAaKRpXZ{1#-IqL$92ft!7X3Q`MY}X&n5&OdRJM8usR@wTJIvbDC zTt3E$yYcMz*DEJuVA3lVKt-Fs^^ENfV7D25Y7?q_Z_Qdz@v^6h2 zYRSXqLI)lI06+jqL_t)}v)ZbP=C7TqYgU1uCunT^DX{nreuof*UH5$nbqwpr!&MZ9tO0ioJLk0fG-Fgj*)SdK51VB@&zC;PDH;f{%W#Z-)Rbk1%Q8;j;)PMl4+UZU%qx z`|rPh$lnQFhRpqSa|r$QeKiw@J~u(2R@sV=lw)qma`36Rx0 zKk%n)wIFwmTUd3aMHlB;+TQh+T)575uYAYWA*KJ!XFhAUf9f`yFma-5yu1r)zrXq` z|6qyz^u=ZNv-L#`M38+VC&j*Y3EVmP5^B@w=iNsq`A79ty=4qnq$n~yG`aalpGnj` z_sNIt$!~nYhRir0QM(so>eM!zo|O|Xg;lL!Io-j0ISh+L)KvCtmh zR$zmpLoFY|LX$(eXtK$Bdjjv}fnZ8Xs_lB*#QyJvqii_gQI%xqcPt{`U0iA_cO0-^ zF5YI-@JRQqOR#n)59UX=N9byEbEN)L)F$70ce{;6{GLZ&2p@`^YCeVlX8KciFiOSX zszDBy(3nLYF0jnJ^%j-8#MZv`uAu}WMD0)P=9@oZX=y&GhYa52)Te=E^wb+G?US!> zvwz)qu8q#>Z52poJmQQ`Pdce)8uPj=eR((*Uw*?0fvLK{1Mrqx!K z`$g^TubY$Z;AS7v*9`*Fs7W4T9li{_jD^{108VQNm^eP}97e2Z3i%%J$hSA1&l-za zyzAw`)61Ih#zUdzFW{r8%7xavf|~6GoQ`?+QQoUP=KR5oSSP#7FDj`KQm-g4{|h>` zYyFn&4q>;Dz3yv^Loky>MsR;cH1i(@U8JByHHR5|3?mos@qerNp>Mi8-3LOy4}|{m z?y4M!kTvX(3$JegB7r-)ys!@_Lroe9#0a3}xb&_nR!B7*QTvK*d3M9U-m}4Q+WRS} zt&UP%MCB!bw095@ZSgH<+r$BBPKSTcR zmZ=Ox6Y?9pwiKZd1jpZ|54Ow3=ioY=TJnwv2rm_uRoZI&m(Rtu?Snhn3!hhFmkv#_ zf808qZC=4nI?&mmAzoRh`mC{0YKu7Xkh<-18ItquMBES-XfkS9^|(>WSud`o7lFsM3CMc9z{SM>dk3<9cOKB`)vPHMw@t-Z1_t zu8@A7t!Q5!nPy)&Z{bWaBqNGndh5BC zfvDX>e$Bt_4%eQsS!;;h)2VB$e*W@G`}+JXHU>_8C*pUd){we;$RCH?8Q9vO5NAoB zSiINz?pSWMtDm=}^ygjQxW~Tm`OjKrW~Tc@V*cG#0oY%#P0h=s3(m@`42*slx~(U2 z=aOE9vfunLJ{lgr`*W6ynj^m75XYSZC%d2rPdJNu7#=wSdySd$3F=n@jFv5>hm-?y za&kgbQ&U45OG_i_!o#Cj=Q5Ow1(9n@a1=7vP_Dy3+ETg~g9tVf32@s&pC6H?3XK{y zDhlt7)t)tc(}Nyx(UohU%e%wl%-NPU5Jda}l-D3|d0YB&XZQy7CM(z18Pq31?ZphX z;+EHh1O~2pZ`rc7{)#%>(dCssCPT`mIsuI@ZD|Y(jjCH55fePRqJ%aC2823}?*FrQ z9)NWf$NC@j-Yr|QW!ZA?xOc$D^b!I|c(lB9{>cj=FQKFnVju}=q?0^)3cZG6FxX(g zz4zXW++=lIvikq`?Y&1_%aXCJE7>w$`{hAEfO9-_ zX`liNVU=BDH$C}*y|TB&Mqqir@Br(|Sxf%Ew)fZ{7h)xUXfp)J_F*Is2! z_M1B~#}I&_t{;jNFOXWy(ztnF9&4k}9(bU4JhYq+?^I9@(pYq9msnp1PGYB=8q+Z7 znaW@F^M`kq+U@Uev~vLU?MT%Mp9Jc9$2V(=pMr=;)|6LTr+r&3d)s`Q{-5V8Grfa7 z`j02%l_+Z)1}XX7Q_RR&b10pyl^H3so5oo<8{rR|HJ(@{SUviuII5t z?GWDJLO*GOtHnw}>;gZ9(p>7kF>k?w$K+cCP?zxIo;8)SG`B@XMe*#jk-{b%$@Kjx zR9YHGB6%$Z=r>{~wPD4I6*S;8JDj$xs!Gxbig}PxY?%8dgE#GAu65&$?Ubepmx?`% z$~;7-9tSMn=bqFv$yDe*Mr!lt%$b9C3Kv=h%$1S{=(?VkTlGd_YWS5U1=SVwcu{Q~ z&|Uuc(+=@Dk`h&S-CTxpEtlRj0#JW--fH{tjMdgBIo9o+t@OO0bR?a3XPMaw=CN6Pi6hIp$h<$4}c{h-mPFS@Mo?aY&VSVEq5OYWZiXr=t?>`cmN#foeg1jcGW*}@8yu+b1x!?L#|s;H60tfA8p~xJk1O4C zcCWUOm2cb3IScL4r=GWSKYJZ6UY#vX(27d6Q-*Y2cm_)6vMCF}UO^hDcb`Tehmt4hs_?rQi=^j-fxcD|5{vd*nA{zXNK6L0%a%fFW zCq@Y!zyK!cYj#K1)NJ|SV*_X}jx*+bWY2PXWUGEz8eV{|qNZM=Dk4$AG$xs=B84Ne zsW8@3uH?oeRNyv-dC5GG0JpxZtZb3rJ)nSG-{@$yP_K5+t@?FBYWQ6MdL{keu~tO2 z8g0!@$Vut+&Z!v41*=d61`*6#m!iJD`_)-CZEvy9*$nqRhIO4TozDO^LRq-4)E*w) z&ThG8QggFY|GfvY>rkJ6=EeE;{MEy3NZ0mOTwZB&H{{w|Yj)dTwgcj&Xa!OEr?WXr zN7U*vniG)c!+Zl79;oa7L+`DyTc@qFK{%gL*&O7(;{|nT zIK*95NRQEgdraOA>%C*4-Ff%j)~QR^p!sYUoeH8qpHat|CY^gc7D=0t^O#?K@iR7X zf@cIrrLEehnfU4qyl`tkFzb6V_aYMUU>`_?QH;ep29E z9~@Kgh|1Acr?Ojl4ixuMVyiivcb{#vmy^~vU zeSBi%vt^~|FzNF+POI{7NcZvnR_oCcwqQA$QMzba@EsCi%h+D?x|io#X8==cz7_l@ zHsoOYRaxln%C{#*C)roNaFy?cApR|9ZTxPQvQ;p+n6hG<-L-NzW;cL1nt^bn@veZm zmhP3qaHY_rZc&4r&OBdpu+l!$DarnM^SPGH{CUH9b68a>M%%Yb7pu4IINpu^SR_Oi2N4g>ZzYJu;*jF>%J~JT8hUIjq*Ic-mh){8A zKp-PSg}={OY5()?S{uaL@jRlC^PuAebuZ3NXl_<9eeuH)IrimALvT}CPiRwx(;!k*B|;A1@59z`QJ6 z;-`;^PYFH2PE<|ukl*=w!U*G#)-WDnbk2R1Dgle*SpT7d=VM@d#$ zNkMQ|q7btSS}F73)Wz2S!D)6E8-+vyW_?nktqc2hr-S50ptsh`3(Q5Aj%3vATLpLw zLVkbsa6gy}6hAHe1-`4N5OP^j~!ylbtf1S46R_@Gmwg>94HNP$}*ZOgV zZ%rI%k6hZ{R-m(LbUpk+@bdusWw*j9oUgCxag>&^dwy=1N$d z#6fzTsfgo8V}$w9b)QO0kB`w#eG48(;)G>f?PoXr$urbh8eyGYm`0P;Y-ag5Jqy{KnkYrA0Wk?Bj$L)9Uyw*B( z>f{~^JR1wjlJe}))z~{Hde4iC-6?JD=It&o0%3t@zVj@81Qu_yT?Nu;QcrID$_1JD#`*p2(aZbWN)VKj zM@plqpl7Zjs}RtG-*MhR%VecMCHo?X-e3gfSW!tjKhsq0a_bt6&BLxuo9yB5f5{?J zyINFSg7XBY`Sex_vT;&JxQX=c;ruiB{%-pD|AW32qJcIc!A%wFggBlF)H7Tqa{;p5O*M$j~p@LDdsl5jq#iMMfH{S z+(u34H})}MY=l<3kK>m}XpgZrt%9QiM^Sin_4`QW%k&xpk6D0ig^ZMIkmk+SjfO-n?CUB@e8w)B8 z4V)XeuB1Uc+V`Sy$R;>Ds>hDa>ijEnT?Ut=I(i5zJF(~J#o)%cJRP%OKhKI zl{k+H!O+mxRkljDwA_cDuy!X*SHSYJ5N_mhyanI?g8TPD>q5Qbm)O;nXj|(F`uxtU zY5Q=@uwi{9sW$`E12bCR3<7Vq_3Kj#=&Bf5vTJ$5sNuukWR3YvbacRi>n}YpSHaiN zq3}N>gbIF--#>y8=HXc^kwz{5nd@T{5~j{xx|9h{_pSPY(*n$)L!ETpi)(zE2Kv8u z4r%YlkkAOO&j_YcCM4a+-!&_UjsiQC!N~>zNd;ig3pcKRnK;1qAsNbYTtj)eehR~C zw~%G0NmmZHzy0ZV?5rkR8g0&^G2zJteXEXT%chO?t-n8S?K9fj{)$ScN>^SRR+^Yi zVBVi~>dWxw{(ZJfy_tu1kl`7i(Xa@8tNe+~05`ZUFo!60cP?sc!oNw;f8^>uazBB>cgpwEvAhv<2{&gZCy zZSaP`T#jQ7?A~MlxcB?EYw^35)_ow$(@NPU6)>kR8wA~;CO?~4x$ZCF{;vV|d!c)U^zgg`S1`*}Q&SG_bwW}Xs1FpV3p4mO*eq~d z`8b_lI%R-<8S^3jib-Povl;Ag8Mtn{9%$nMo)ltzB(NCX2M=&y~)+Z@l7K*`c zX)=DAFtD55Hay4ed))ywu3U+*ACz+s+P!Ocd-|oHI;~@7RZ_CP z70^BL=>M5zc4NhwRv4;ZJ33doBMGu6z_u7qb?0X#*^e(Dg3X*y%5_~*gB0JMI@P}Z z^{?6Ji?6XkSN^wUckJNaaGRF|ybHfOVAZYPzO&p8AFhTLpE*gxc;(^Fo2w3pqW;or z5ZW27jIylHi7JiIIW5sH>78beuiIyRp7_w_v!-5YXh79a1+Vw7AJf||%SpAR82c;PhsiA`7Ygb=LCV_{K`>{^$pG$BT1q-p1W_<*nK6f&2}`!vP;rE#$4aU2rdC5NNH^v=HK_#iT_o(s08G*9e+V?V0z-Q zoPJ+;U|xX1vb!2KlV%j01LXZXfi#SPq3P{oSrPRlTHVo3E1tLq}ib`B|r(En&qN;spK zQOqQNDy4n{IhVsA{88sEPzeVlZyZ0+F71+LtJyBIy$r74lmt4b-AS2t{skA=_rCW% zd-Rb<9GJsbPQpB1Xg8h@%PUT zx3suur=D)8YFFILnHIR5?LjZR@*11h|0kQQJoq{c&x_oh%TPegK7KNwd1x*iD zaj=NQ86?y7QV$?2L0eC&PnO$N6`KSwRlBY~DJb$9mx9?1#8U{szfWJj1Ziy>X&r`; zC?b;gH{1mB%-Di1L26?f(U&#<4NjQK;_oB0)esq!hdXl5>x7Z!2+1IaMuYJsW zNiwIRdQ)lM8gp0aB#yCqm_~GLrDTs_!Tu~2T|q;}4One8b9c+;EnYl#&z?OMJQs%^ z4@2?VsSQ7PXmaQy_ugB3^X+%68s-3zNPeQ-&*!I>zF?%TK0zJ0Bvb@9S24lJ zoH~^%REUYF5c_sWv8Oie#*5y9&klXxa8{C7V$ z%toeazUch7Ki>RpdiLWLT-TkkN$83fy_tpO_Uh_A*g;fV40#P)!Jv*Qi2>Ut z_N)Ib&U~bukrA9?CHbHQMJ0#fI$+s0ADlHi25^P&+S9-Ki7ov16!N=jeb5mC5&TFulS~Y>?&fB(a<2BbLbpF%^uTN@~0?@pN)a;{&4I4^@%>=W% z%IJZ_WoIO=aGru-61Kywd7s}u#!K9LRAL;Jn*!7A508qPK6mlrl{`}r{pusyM37P3 zj2Sb+FX@JB`J7|Kj?T0poI4bwb*FkP~Vt!Ziz<-(8+b$Z|#c{C!$LrNRFqfU@AAbL!y+3Q7 zrCjqJTOX4O*RvF?W15TGdtW~U?#ipwuHixF>eCtW(ZqEF^J8aRv z9<#0!JusKvq&}#KT}d)Z3InP=8IkLV@PQe`?b8$x5TKfbFsdX7*!*x+`=cAy2E4(6%lo&UdhSUW~L} zO0)@7Od4r!<@gMUl9RWRSl5B>Gews_$}JCL7mRh5;fZx6eJUHk07 zfddv38|&P~df-@%#JOz#M>eDPH5T8cFH#{+{TlT4s<0x`>x*_^1ybfM{WI;}&yI8q z>Gs9-zwgDV?_YW475naY|JR1!@nda>zE20AlSd%9+eW1DRen`^uk&^$mY^RN2K3r?j6)&GSwS63A z-wXftkMleQsop=F5NS;YHmq}N(6xqN>-S~kNLzR1r|&Za<~Z}YsZU3{Ws+~3*=iUn za0;Wz$WQP=F%$KgKX230Ic2D=WVJMH|IVFQvwzptz4e@RKJOyt%#qBGUYf>s^scYn zNHzYE3atdRC0;~xj^wvCIEmys%Ki5B(b^vg48_S8np1dSGoZhaXl642UCkj!Uf4E< z$P=3yy$-R7Z|_szZ*IB!E`6R}n%$+#D@&Fv$&<5}UAuNwqjjy?wP(-%E?v58bC#_k zAu^y#R6NkKkx1u~mYgMrpn4l&tHDq&s;#yI9dhizr0-ct{w_<4uCwzeUTAqmrFO?1ci4o96RmIGzIM%LK4UlDaD(M^ z>t;Bas3%LHmzt7dSAFxxR+HA-;^-tBjz?5WCA=pvUyf9o&b;=oKR3$cQBFIp2{hG< zDZt~fc!;fDv&N7fY{1ulYTG;XuoPHdO9CczO+z5|OHTgkm*lH&R_;1rH%(n*T~lzC zD#HZgO!Z6XURioP^U-?d&NYV*u?x5OwktTAn|yal*e!0O)@RX*{yBGp15#-Nq!oxE zPZ#I3a{~j}1;{9Evm|NK?SLlY%I6o^Z`l_05P4OfEtXZ8jb@tRw_|c_Gd}gx0poJM zJ66Xc`gObk84mA{T? zrlNLj-fR!w`3>7V>lN!XhB<#A3kIE+c_GleQQZ=Bp1S{##u|a3MQtdTfTN}=G zm=2yrz;n^89gd|8W0}v6BJG0^u!!gX$N4Tast=$kQ2%{UfD3P_idDjrq7}b_rc<5c znA+;<|Hv(?oV^4%jtS9krd5~3R98<0Gd2-{8j7TKtr2>N>t7IuQ!}wo$9?!O`J*Bh zM))n@_488~T3SL6ktL!ebBq|SrgYz=g0+rZ191(nWMl!))NbYXLvY`#Vk{~x31;Dq z&G=!-AjTX*C4?%q&US$7L%5Y7mjecRPVo>@WI zAt}8fFg7Qb4oDOM9a;B|>Fif-_9CE`#m;8hC%Z7YgVkpCv#(R@yvU8k778Mg~+qULfB&c%1q)8SP73Jow)&E4sCfHxr=2{_fv1hNF+x032C}vOe zFk34C1!z`%W1^KH^)BC;Z~akOt8*35!qZN@uVr-i-1%KwmO8-F$6ZOBH5N}C`vLSH zo!8s4(KOV5!?yx1f$le6nCJFlPr+bLhHr&PsNcfEKU2~-EkFYYg@Qwe?SH1Ov^?E} z)>^*3c2KrOLZfcgbCkPgt{-PFzg@E3y5hy|tvw}nc~-Jzr6xG7hg0)2PyOWbU6w&v zYTzO@xz4hO=)$Hm&)~~_OYQ+1-Z>3Jx@dnIA=al;s=ZAetj76I1}_VokGFGCPV|0P z1{7;qDKMselHGUJXiHCs#RyIw^}Nzoj++-q#tkAQl@~E$sV|@AMSDuV7T+kyo$KZm z%k81tZnXSuYpi|WF?P7bZ_VB~W#F$9oHgIG6piU#AUvMJ;1qI*hXq{ZQ-88yD&%MEH}&qU2Wwr8(iPa&zxayQlcJtH%-h$s^HRvOz#(%AwHvq`s@ z@B2naMUBtw(4l{a%*_ADSWbh3f&%TJ`soVC+vZNDR6-_9m=MNDs~RxBo`Qdg$`7HE zE2zv!3^Z6mTzi;GeTm9>go@fiBG?y&#PYllF?POu_~zy1sZzuYKJGmkN=~zH1m=?& z0e?LyeD9ZTzb7>@HEeW6d7TD1Y8+=wP^C%r4j$nJRsBHl0(|16O3m8#$!d=)(K>s1 z!yfA>P)AZPjZd?(tiBeK+|5D{ms?%lI;$>VO7P$wi>S=Env$K?ci>=q@s-!?GOW(C zv$Gv>iZ?Yr+LB5S*;||Q0k$K3$ZNfHDVZuStFmb~jIzU(RrZtVtL@GU2KZIy0#Oak zeEEIunWyb%KmV1D_~w1Kx-Q%@Lu#EPmdN-h``Nh|?bXkVGcPC#_)Z&{{C&l4>!0Y` zhdogZg8uhABf6m?XQwxROHdl~H}A3E&E9N-SojF#r|(^nC~I zPpfv>h3WCuC0(nHuwcT+`apcNKe=GLb!C}a5%0SExTj73Q7MV><@;V>J~S=CeslF` z1}+JdZSXw7c@&5y@na!iF5aX_+GL>bl;yBRqQO`>cc%U6rcv}B@s`}VmsPMdEtC}= z#|X-`r}j=}#Pkun+U+I2C4l3qg^L$&5Qt)|wljCno*lYIrM-1vZ0xq;@bDe6At8Hl z^is^IwF??M<9&JC`H>s3*3v4)MQLImwTJICpOBU+xOOwOJbL#UXA~eZj}`k zgGu8#z`PdRkdYg(;DTrk;eK&OM#fe^y@vEs0rO-A7nPhBLBo4B)zuj>(b1Pe|D~O> zvR+xaa%HKedHwj|kLt`nwG^O-pkfSt>#etH0Q0XRZU0N136)m?kVi>0=Xx>4`)6wH z1AeFnOokzFA5^z4CLv)q0|AW=Rhh00o@rI_Ka-Kt@4dI~>v!C@xU8(^hUnPv)aoh+ z>Md0#r=$qgVQSc}ii`@Tm`;3-r$8#EW~^T}pbl8R%2uWw1d_~!AI1Y(Nm`a20@$mv zhgx+en|Y-7v#JgQxi`Y1k=(bhS!PK|NjB+%3*20~=I135c1Vh|Kd;>BHh_U)9fpb# zm5)4t=9go-A)c6B`NUj%?uLY-#V5|&P^qnKSyKPIsp?aYH z`-f|6cHTjjk%dW2LSnD7aUJ5F^A`~5J`O>lJk>GcA~A2qnUJ92U)CTk7V%l?1nt+ zkEQ%>-qj%ESV6tE=FHdQIA$1N{yV_D6YQ6x}Cdp*;7+jr#+0@!QY9} z1Q72tsQUMye!m&D|5rL^X1=jx`SN{!iJ!-xju3S4kfq^u%o+Y|&6--k{M!Kd!&IP3 zvj+7_1k5O)egcMjn)~@MK_bmI7LY#>8WAyvjqTS`p<)8BHJ_1{AuxYo_3FsWyR|R< z;_dhB3y%o>Ty<5g@~r$iM(2D+1Yt@AJ-618Yss}b@nB_8fJNhWfYLv^gJsgCys~}| z^XIImrYNEgQ@4e85#!LHhvw5$w+%3kdz>owauN`Q7cNv3*=UNg5WftD~*N0Pnt^ z++_>9o@ep>Mp_A*U_^62o-)`EpnrEnSGOIjKGZ-UjQDmJ9kQ!mT;SGkC-Ymo%Lx#* z`O9Eb+Ipq6J-@`h@a}5s2smp=m;k+)^wV(zatX`VRNkUTWyHyyx3XUS zpDS{ij}K>eJUq|=%+;vvFX@-%Gz=PG6yTNa)wTO98);nm_BsibOf>Y+n_4&2;~Ee= z$?kKd1;sX)Id82^?}@tnNy>i<2729jF5^M(dsj&OQQa#poi*pa0prDS9g^%jnkx%35-Ogr9=Ibcg-d3Vc)(gNzWLvkeM-OqlulHBEuDGE@J0L;ZU zqH90r#qgul@iL<82tH`c>ZV}<`9U2sGy8g=?gwLa7n&RQSSAGHxpK4yTWPO{i9dzM z?cuRx8qw=@RaGxAh`0f5!w))TW-dhAu%71SGCevqQ zq;Ln7^;b_?p(I04i%g+|`2ij9yQoXTr~o+;sN;HN6}zJ$mFM~iQT?kT&*%zR)30Ck zWYkm_bqs&za9P!}7|4-tH=^|y!Hh zmX1WMb>xvaA&fw3536NPmiJ<+Dl9)K-AcQSb%p%OtFO`D_`uj>x-X3CWj(MkUyDSZ z2m(kotmRbun5TYt<33rb+0A2mx$mXkR!|%Y)83hCe|!90>p$uO=jkjO8lyOhW#9$? z_wJ~Eh-r8R2-=ZMwEp?MmA2);AsYhN<^iM%0%_@p1==zY3k>r&?!ok^(h?A%)mW-& zE|Gxf!@G(sw?KlHweIcVnJm=h;`NU!_t*tbeq_tG?{{m{U%GT46Ymwwztd%>2>MY8~0Qh7-m1#|J0)a+4cpGph!?fF04XMg+tE!OY636AHg z4&$3$*6jZJ_SM!W{*{!j&8}ilE4`*&&18&*bSz~|U{De25Sw|2eycjY!?Yw>efaz( zOXkJJ#l3|-eGc*O;)P23_-F}ld=yIrdpZ9b;Jpi@I602VhX%QL1royk3+#BActbT8 zF10z=UQn7|y2kT$NaRy#6&r{RgXbiiB+m3lV}`G1^?;G$$6#h_}>d#3wu8l*w32 z60>@<6h(y_)(2Qd);rS&%uZ84+LF$-zxl84{gK6e?bF>2b}tGAvIe2xt&Fpf#hE4N)y#R@5>rpAJ5;K z=kB_1gy{Z|FUK=H*K4v)+9oL-2QO2%7T5=CcDlDT-Ec`~1}aF)K3%E*<3>)RZwGO7 zhmKzH5q9;{3+?aIm)jNiX}>Hd#nzyD7l12hz>$JBL6ZC0qB8s8=$>}lr6Vng_0uh3 z0H<6lpVEGa=TuETN<>ZD2=e?E#+*2F-L`zzhxWTKU1}RXdewRX=H*zj`$DsG6^RzB z5{+tiK=8eBaF9n7UqlUE?AK_>eu-F@F2I@)Zpb#)Ygd;6pTZeWm)!`QkJJ;~|P+s2dnX;6JxV2?^Kt^Z+S_J_R3x74q zZW;Jm*MPtc3V1dP2QU&lH5K6E<(`rfcu8G8Wk-SSXD(cUPSTx4VKLJMDtP|80f4jy z`}9#lq}_{@UWgYt85y3Fh1WQ4W5EOfHysqSH?NM;*M@bBw{K5fYma=m+D5UoO$2M> z6%97pQFihQw;hoG<0XUb=JN;3hiFjuH4mgzc@%w;%w?GAm0jRFr}MRwR->64uRLti zpL^1NarFcXk4>}=EK3uZe}X0Z31yA4lFGZ1dFP(;ZuA|4B#}yd% z&P=s`PwHd4@z&<_kbHObIoYn4!a~>AoA~`?zxMKTwyWP2mYK@@e3dfj>oa5>Q7quZ zxUDa1w!QGw?6vmX#sfBxx$Lb-x^iOe^@0LH8RAK_%>vL}OvIOueZULrQEZ`ZS9yBx z>-iz*?ao~L;sa&283VY$AnCr$0lJIHeX9;YKG^#rVK3QRYHzV|i$c!~X2mdyllni7 z3R5~w1wE7ILR+XxS|6AZZ+~Iew!g34V_6I?M7TE6*#h0EXf~ez+*rGGST6`?hd7Qv zjYv~zoa}vpxdxCLNFFN>CmVlDeCHb+!YZiP59H?BGrzgV{(jq6t>?MtJ0A(!F%4;& z3tk&>(D&6~Gv{-QvQ)aD0D10|@8Peiq@ZWGNE&n1E?n9|QA&Kj>Sf;b@eqA8GwHu&02*>3=gg$pUxo z<%K(8g$If2v!a3b&p3Cn&8Z~KnXAt+z+CPFMMqaq^2j%xN;JxYlkaY95-y8 z!C(l*srgt*>M1xt`Oc*y>_Lp>b~C5G1yJjVv?vR5wr{d<4CYD`+gau<_uJ#Y`KImM zxl0w|0LLr%Ye)97ulCEZ1$l?;i#cgFsH<$BtL={8zi8b@kF$K%rw8Us z#%%|X!Y{|jE(7N<3UfE@v0G+rupCtF@^7A?B;6C=A2lwm1CpqohRWRwt5{a{5#Za4 z!GR3Z0@IO$mUy8T@h#pi6BW(5``*ytq(Vk<{tuKl9g?t%09K2Ou$$*>c2asz1~Hdl z@OBUlf*VvQqGJTbqw_4&21Evp zEfEx}v_RgLsG{$*50r=ev4^uNgOy*lYTg|C{Wq?$S%3YV4LEP2RaaJ6z42SiWyXOz zgDh=T`jNbQ3P=&&oTYd(lNEdGQ1!Tqml2(WJbq6H#CLF3hy^#sprJ@&F8pc+E?)(l z?}3)9d8Y4GosBV0ecY1P88E!XpkOh4KM84lu%!2x^779CdXvy5-%1=N3~NI2*W)%9 zngi%6Ji~b5CJJqp)Ee~7Wz3ZirIVB4f!qY{p)lXah2MP5U2xBl)u&t*Rdy! zRPRc&Dl`0Z#f8;#5>mqDUQq_tXIv1@l+fuOY#~rrfeF+r0aE5m zl0g!EvAz1z3ufnEX>sw1R)lo!ft4DH17_6CqcWL4XU_P*!E*clRQzrLzFJ<^m?}n% z6D)V{g!wJpwOiVI>rl6oAZtB+Ms2*}1*Ij?L(=;@TMyvH?qgr{hp{clV}PSH^^Vv0 zl?2sbeI!liAF|gLZbEAxY8PXDF15bOz-zc%HW?I;^qATkU7CV{Nd22ge;0B5;{1Mg z7shWLQ?jG#Obl9*96S9rS${*9+Ue}_K)8v;HZepU4+#q~+g#VR@!9BTo#dlxKO z@_uA>^$bQcI}rveaQirhy+0-hn3)g142^Glr0WZOpVfcw|5<U?+#mwWHx*6a3r42KG=(PaI* zK#S)U9GwBtwfW^X4Ub+jw43~&1$Hl9)dsWlO{#XF^oZbn*9~CR1~BjP-9`PgZChT6 zJsYqYWILy5r{h^6>EEzjfkf!zJ9KA{Gw-jnNz>~A0OFLq~)HdzV{P{ zm#)6kp5|L*@JE^jw_RfE+4N=12fv0f zNEnHk|CCva7w_P`BWO4>s4be<$|n@iNjz2> z>aaA2oEjfvU%qg-{rlRH)(c5i26OH4kXA{@nT5BlJwvaxmnCfzy!l-=YjnRa1B?ac z?iulUo#2$$|6?}cQVWTWXEPCH#P`0KUMiOY&7b38ZD5ylx90oa*;{QGV7?15Z>$k| z-&Lw}1z=?fuCQn;bLVW=ItaS=Gsy4&**)ZsQgu6+VS-`gb;SD_RxwP6mUFZ5X|6V|m=< zb9>vbuN}|yM20hbldphaFH7^K?E9V*a z#ZJB63Sf?J7PqgtXenoqS|Oh1p0-3~Y>4Bgni`DJ!+yln>a$uxCNOV?v0Px41zu}V zVSXcP6Tl*wPv_Qq91&; z2X1UeLSIRt$rGG{-cf;i?OpfIh7%|Of%cw>C4veGAYe-}jVEnKqDJ3eSZZ&~TV=mlzLQDd2+Kk3oDdyjD`rfy<&&?s zL4$|ZOX;F1u)%oJ`_3KPZ2)ROmA~j!H(K;{-?LKIorfcFNjg`MR4cXqR@R1phwVyl zoixyf{c9#m#WC0;{XBqPbJPvaKm zw5|Tqx%MQkWq*Y?xNLmS2hP{)J5ORyAg`izGlLTY$gk>>W?voO&w6KNXunN2_oh>oH;kYWD zIxeO{&n)OF6_9g|qq4I2q&L+%fa~^}K9ZzfSyi=^-=CwO-zrJHTeohto-NhUNu2Z@ zv6l`X$b%(QMvNE)4|Rezc0+bcoli7!hgtdvx|5g>9ss!S_3XMNt<#=shww9wT08C*o%|D#?}Kh)~?SORP2?g z*gv*=2v)j{#G}nw!mtl{hI0Hj&tm~NSDdmq=h_>uB{Dee56TarBKza?r4#GKwbklX z^A;_7Tk%R-mo71&WQ5d>EtDT6)1u)-r92fz9bg`R=+L22RO(JfYOgi`a}~a;fEK0vQxdtCTs7~$qWJ?ArChaGHsRBnn7SmEKlnqC$Ct8KVFt@ugzHCOPY5z2=K7U1M>q14%iDXzGOCZ1Qz?%&ZVly z{5nhe&LjRbmVmvyDbIepVV`9I$Xebd$<~P=O=M1GQH+`z@tWEWwb|?G!x#ia@F37h zPQ%YtW~F?V!azb^<5mLX3Y8ee$@*PZ@0eh@dx3vEdiP1vPL8%mm+b%~0dceerGU8v zd!;LByc?je0l`|PE2{Z@&!}$p#DAS?->HB8sERG9_TwTe$*&aPOtfQ}@N zvW(@PMz`8EEs_^vb?{0B{rp^bUmIzC2{4x{)&1d-k#C`be}HS}fsGvtkH@o|UO$l_ zI&~<>6>Kqhvk0212M-?1e9#F8#YHw8sMjr8v}hIiDjhRySRXmC(I5eU-x41m-(CXi z3CDgKe4sXQyn(uRU>?q*|8^AWGJN)Gdprd%9I7akIhcnL!?1*EDubAo_O9ke=Xb=YFAmSQZg?8=#Mj*gDDi^iN|f4_c${bF#2 zbxMk~|9X9k&0Dp}fxl?*=n%Lo$P?U}lmUp-$@G3TMr9l!^?Ki^hUOL?vJ5hJh?`QS zdvPhAhC#iKjN90-$F4=1KfnemFUr*`0|RZ=qNQfX`d-YUbl9laW^Uf^qA&9MagT&T zIiqQeYXsU-t!D$+3sKp>wPn8zV+q;;Kwg8BE~xUgh;n5?8JdF{`|BG%5fL#cAaZMo>aC(Cq-lV@W|=DbDEZ-g|ZAy>&grE?X`EFdc^L(V2G`G=P4UA z@j~1cSF_oQ)>^b~5DOuFhe;wQNFtBmI+|;3hbn-^MO|1VzvD1YO9AMUv4eP_yrQBf zlI(3f|DRDKN6s8KWJt%i-+sHozx=B)whBhZJRmj%$bCI~_ADjRlTAx3q;Eqcj7B9S z?8;9~?Wj3=K)u{ApoVMiJs5w;B&0knP6fIyL|aTXz2S0kzC-+Hkv zVBPy=PtCWX?c>~h_py2__3jc;+?n}ydU@w2wi=1w*zf!&1GDhio?}__U{$$JEoocy>-`a+q6B`ZajC8B|B?# zQQ@28_Zr5V3)kCKFD|v7?XYsEBbNobyr%`y>-*~XsFs5k8Mdv)qWzu`*@pSC{TAc2 zj%=hN=P_QoO~t0?T5}|gz$X0A&;9Cn=I_hxvrjLu5gik34XS%jN2+&4n>T4Hx;1~V zAj7$R%!OmwX4%Z6e;Xhl-KUFnNK2-#$89T8y2?w}{!iR@E%C=k5j3G*ke5LPbwZ;3 z^a(!8b?IrsF|w2a=JvHKZThp1+q8%OY@Pb{vc!yRpL$($}2)CL;fXgs;(pA;`iwq>ak2xx_>ElF~qhN`s>TpYa{hHj&&iGN(-z%JXVrtpEXhf zG^mip%y&r@&;EolhxSxG92+0;MfwM;tfcJY3c zJJFiZ`_4x?dgb%e;_M*H&6Z<9o&-Q2Z(J%P&C~BJt*}30ke7(c_p5KLw4oj1YyvQ4YVa{G6rr^^B0oWotgs|kR#n+*D=X1W>>Uxx_7da#000VuM&_;0_A&^QA-Bp z*Y&tuXj$6nI$!P^*Z|}a_#uGieY>{XoGCBc>p#8SqS}RG^fnG=hL7<0=F}O@FE=#f zE)bm?NKk2R40=2j;*8>$4%p7eUzxvn@eF?*>RT20Q9<*{fL#{q_O3k40^Bj93!RB& z_!hk^ConKWSNyb3(|okJ!G!NxPr?e`-J_{%j{P`dY8$Tzc%l)RC9TWe ztF^MGGNdeBA!+Ve;G%&=GKbFLazp0__19SfclFCFEIKvbdLUtMXYN^a%h!seeZ}H6 zI(>DlAVAiYDw~aBcepOT{1jV;lEdHtXT{5@#wINk&4H$GL+@ z*&CSu*0!)Bc0#*2ySQ7r4eQ>~vNPLRR8*ukUqd3t%Nuce<#DXEoBPZwRTaI)7Fw33 z-TI_*Kb?Z}@&-W>k=hXt+Yh$SUHZ{9d-G3|ZST6xmVq>|_ zmkQ~QRwmpqP1BIq&*L~$p^YIGmS($!H-pM4rnY=oja-v%y{3TXsg8B__nPferK3plFr6 z%5mKqK(1_vZk3PDa^Wh!{~Exle0A@12pZ_gdOac{iurKn^bZu+ngz3L+Q0s0Gp4;` zBS(z1$hdglnsU`#^CfaG+pY+x3xwp_zY_Cw;_k=!O8V%rDkx}Ezx+wlZ#95&7*hH( zxK_PMeY$_s5-**GXOeI%D7JI~VXp&e4Q5oJ1ebsZZb=2ep2}dMiv7yvHPovzeeYfS zA~-$&*3;Pv_5eXG0TKR5!DS9m?+d8kMQv2G*~)`+7cXAteJDnl@WBVQlYN!d7Anjm z{fs!%phDip{r2TO@qbJ}TECSB{s7W?d}%SdQEJ3Lnm?ZUKCR0{8F0M`dBQf!Iifig zoxE3&0(Q|O4SXc#-|>KYhuBbCxg*cM^V(vYwGU%ljY8zP4a+`xQqz{F-mqVf5I(6Q z{p0M9k4yJE@T}{=;L)J74wXawHCF;)fud-W5@8DfUd7zGlfso43;u6LojwMi?Y%pt zxq0!pI94x(GB?g^6}&4ovX;4NiJv}ct)RTfie0|`%b~@~TWiwLR9*$-swg>z2}eUy zT3Bpr7S6U0U;4MrX1BfUUfnFUa}TRHRO;NeYTbD-1F6?iACk^xymmLVY>=p}G?)*c zK+$~#L%VqDPONpT#OtMeboYZjXEDbytr{|!q&Vt<-3fhlby=jj+f%`lwsRB9DV8{; z;Z^lPUaPVa$$ff)3io(|s|2ar6XN0?go(Zh6TJ^N{DI1Qmk8=Bmg`UeJ~TZ&y$)-5 z-9H`LAY`2R!SgJqoW8TSJSVg^{C6?&;nyNW*x_;?u=D6x4ZKYilE4uBbf7L(yh3X; zo7aFmM%R)p0cjF0REw}|-jP>md-IBI`JMvXaG=zt;b6wtPu+EE>Vt(;Sc&T+fo3v4 zrDoUeY#t!{*U4dK}Gk#o(@>5GHzzmg?ZKCP|D^4TjWoz@CoUNui%O2!z2xFTw;^qWkF@^wH?9fgI5V-v3Vex8?ju> zDE{Ag4Qh|0fepO;9K*#aSvfh21D3QYju0wS>()a6_*&FC8Zn+}mFShBwigV^%gYPV zRD?3GeCw>h8s}!`yJ36@2?=!p11BX@HyJaPew4ec;C17jT@|0FA>ALJ6w#*=EUBuj z!L$~SP<5ZEj??F-eeeoZ>UYg&3j}pfM}6V9QZBsoe3*(u7NqhB2vVP312V}2*nyHm zcK8r`l>_ekiVj*)Da)Dg9=9Hl->r4xLT7dFibR>vwSxcT`@vtA7=Lv3#8;is0t{wp z+aT7fcS($~)TCJJm7Zv6NpY5(7-tEwF;<7fs2*6_cB#@e2=Nk7bPk9(w^mToAg6^V&SU_&92Ig@|f13xNCO1joXM!LPwb&qhw zcfMO^|2C%tyaN5iI5hg&xN&0wBdtSH$;h)#esc0}S<>*G2&_5*8>%N{7R)sr_00?z zX<5K%%*C%b)~s0*mR(U1vOg)QmUujXcZ~3fM6~V;DjNsVTefTo6_{pcXS?#+v15m$ zS0F7=^-njrR@1s9)~6Oqg9+6@qPVy?6sHSz*=66cf`XWu$&)8m%hO!Z-s8o0{UfEcDbJ7bjXz~!4=fOO9GFCNY9{;@rO3P zu+qa7xcsc*sJ5c=3M*#I(Zhhdyx7%Bu_8%Z@#t_Ur$GYoF<_9-^=OtyCE#jRd#1-= zyv9hvV(~~88x`RIQuE-Fbjz8Cu4Ilt+rcUb|3%ZO3Mv_;E)rCKlFAQ4qoy%L+o=o= zR4FH|5to}*6(xhy?f@kC1^EluOR0!egHlZvb`eEs9_xiLCdrh z($1!z)wappc5~s9t{Y0&)%3A!6jr;FcYeS$=t3fkK-})Zi|rs6vyHlPMg!L)t=}M` z{BboluUEf}huH@EoBp&!n~oC=syX-FxN+kWz=kRWIrTN}9-E+a<~j|!x6=P> zX_!L&H@=@||DJ3KoKUkFKYn~lO?i11oOH;WCt(fZ7&ztr@UXB+Frn+9v^x(3ZG=H* z(Gq4eT3pPt4f4X%2%!#)kZc{pN|J9Nb?TFoVXVNU0`qx1t$49MQyy_sj6;(5k9vhSBlFlL%_+979L!JTbXzS|kaDGw-DlGLJ%(t{R@s z@&JdQ5_UjhZ(y#~Kub6mb1tPplrxwpg;r@=n+#9R*c$zni8->vs(o`qQMahL*dMZU z*q7O7JpwnfN=wh$yy2WG*i{-eCiQzfXekjzLH7e0z}-AWe!R6nZ3-`*xABBtiRQ&Pi zkf%4TEwA9LJgVAG6>Yxq!fpU7w+OTy)zFo09*f$plrc%b$ce zw6=8^ml=oa-z348#l>}}k1U!qXU=wi0`=dVOoC^>@s)rI@q~gZ<4|nuAk;c3aJ`l* zNabz-`wcMA4`8Nq^c&`YkLa{G13rlFwvifMiA1oR8!G|04V}7m+txruuTPr_9wKt(P%96K^U1lD^%$`g2s=-WRQZ_Gr?;LqIO0 zH|5JUkk%*1rRcRWsnx^L@>CqIwB1`a+KSmTZ0;-n#(I5$bwMIe!Pre|^{R5l(axEU z%gxD9?NfDA1c3RS6u>?P2JsBuE9qM+5Q4M;N=GX53R1X=vA_CJfqGyYt)jn;K;tq6 z$@Cccd$D@-D z_1_EF5!%!jR(pLCgO6>oD)qF9WvX?tMId7!%*#uMP@_R3F?yZf8 z$m7*=Bxk8?PIjc|(wj^pn19)25xKd!;gX=ks;g6K!o!nECmGsPpf?5DQs`GxLumlW z%ISQlBL&)Gp(&no95hCGdGO9Ifq*|=$DmtHUAssAhC0Ntg zF9xNlGA9%g$9|q~LLIo00mu5zz51_9j%vU9i6?#|)t;@XtLwS2UVQ(>(bh{z%2{4Za%Yuf^$AALL4jehILYS3t3RX;HClUA@efygwB~w?A9So-%7c zXn@71bzu6VntjU;Ino-oyOW^To4eLJ@}KejF8U(pypMhK*D@pdPZKO~+G#M2Pe#WJ-F5W;He>b`)K z{!-uo;Uw~R)F*yWk3l0^<7sOL4H^_jAD;?6`GC5cZ$Ik8vp0^r1iUtO+#m2_ecNM% zl|=oI20^;Ql!pPl?V*v8YZw`d;V$9cS9y6H3?wzP2XH=cM#bYK2tRCzDFyV7VSQ>~ zdAKa*d(E?GLwmW%KFoWP_REP(@o1F5XLXt;)ZCCHVL9Sv|A~-j&XSO2jV8%iDsShP zn6p&O0%rxVPJ>=wXbgnCd)}EhiIUj)L%iytFa6oqY2sIhMOZp}Mc{F+ zs@}NgBvpzH={qIAy^7Mjbu5UkrEUUt1+U`#KYcgP{ta3QsC+$!RHOoO?cOFJKUMSO zGFTIkhaz2vVf&+|kG}@X-@BXn@p(4)jTdYY^W!P(<(1NNgoQ_AM94Q%t*Z^GO$8%R z6za3o@u--7hLpalE-WmGYu7&_WP}33dR_&nmxBH=0P^*u|5=C-EqYIVp3h3XrsmZJ z3m0DNS8;1FAVS^99^q>k9qSsN?@Q6kG)SA0Iv8QtpI!MiWMs~B^%VVfHAau#_&Q)ywL z?b@`?R?eAWbDsS_+qikB^}^SBe7j7H+%QPw5SX`6(C9$x$>rGPVan^R+R)J5w6k5% z7{+GusUUSnertL!5-JV>=voVRF6H}806EzY$m)k(pUFFum;!xYlDT}W>pq9e08^s9 zD1_0ZQ$`#Ta(+M@0~!N$(hsL!_|yD_3;$Ph@_u!AP>TUjk0y;6k}5|7D zrc7(3BQO`J3)}_r%8$o*x;`6DyaZIXDm%@EOO-A+t>p{|W%MSATs)*kt@m0Z)Jx+G zUS+AS^vm!Kke9o|snr)xf73pE{wa%rm@XqnT2x#DbK@AgvAnRAv{FtYW)6G~#C;bq zUeusmLXfUI0lIydSMQ`}G{ij0uN&uE(T?^86p^DsB&A2tHfJ#ZeSzNdOX_OpsuG1d zZF5=`5ixr>fDrsvh?Z8iKLVG&f*?z@wl9FP%^Z3<+sfRn_mV zSmB$?2uk#y-blR4ZcYg(4jqn3sGxTAlYQX4EI1?5uQt-!C|PSC5;Fnh@jUMYfcN3M zKD^h32r{@I>5zk)et>spek%Bw@M8wmysUCKJp&IlvxeqFh*AiYyb}0 z?OFVu<(61g|6xO{_n1kc-3AT~>)f-CrDbMWVoE#b9ZR4{U-I#C9ePWbQT3Xqc@TwS zCgnlu_bAj$0(NcDs^7ZL|Fhu)Ng3@u-C25`OaEzME zPryhz7&M%5ai~y(71U$2ifnYFmpIu+;Y#$SN|np-!HK#nDPjaxUqbHf^2w{VWFnDM4%~ z=!sJL4YioWWa^!zeMscYU$>S--dMf6IdA~=tAOW|0g|=eRT1l;M9sAENlit?aOTT; zaZ?g)IsM{IFz*-W3%%XbHnI9>n0$aVh)YqpJ{kaxFF$y25`C|W{c3I4;GPS%RhIiSAG9-=bRGr~QEfv!UHIYLG zZd$X+$_I?#*m;Zrf(+u^JTcFX4(hd=gwu~-4;(0beAh9V@~6D{O!s}$3VOxMx2_@S zh@ZM~hX#5n8RtDm*c>SA$8$rg!26J_56c&e5n_pyV@<*2f0wOoC2LZVWnN=lcsE@0PM zcK!B(FDJgr3ApV&J|oY{vzJ%ztc)UryS3vL0C}J^9{5hM3q-u+<0nl)BZcGpLK3#4 zr?SdQ3J%!LP3r;i*|zN6S8dax#g;^GlH7j~D)mHXKca!B)as7U8@AP86a2~X`JB2` zS3d`E$^ncr(-6?SB6aL1n=i)V^kjH|<4^IK{# zXGP;^pfA&z7Uo(H46jZ>@5;%d4EZ!Xk%Jhm^-|Zu&pPE@gA#Y}ng$%ssSNMM@IoB# z%IaQA;$(@RKs>AnXMYc3C0^IraNlRlL*J=ug4ATEAOOUOTk1fej}9~(HRfSExNo~{ z-@ezRe(5!OwDlc#p>^$lj&u`*`>LkK63Y%a@lQcjk`a~3zaY%(Px9)>ST6XXL*1mIBOX-kdaS2Jz`<7Cb8IEUQr($%2 zg;OsriL@hvDp(kxa@S~0pe}%q1i|#1|IqGxNAD`=?5`*H`*i=~(e2T!vzI3|TB*=w z;3l`RI(w~4?|9z?JC6e2I-0r$l2W{dM>3ccEvN;`N{Z?Z?8&v<%^Pgv()qUPz1M8% z>h&Q4@lJyWTWm_2TQgo&!Rl5m;xPDCuG$)ZG)b5SU ze=D`XptNdTKk8p+4lR4>KZZi=_ah@Bl5rb617|ZA1b)=u+Qp%JqlXWlfE3e}KBfQ( z=u339*Ghi!zhBMNX$14`PW3Lqths(c8qjg}XAn!ErP^a-B!*0;{d0&>JZ8~*&eAce zwl*2vlN`LHaxaM76fU%K`2 zsFQQ))Zg)6=-tMG|9gE$aCuiA{CUzkdNbAVLDYo+-qGk2lMrdMXsCS{R*XVA z=^|j~0|9%ZB>0kOoxh?*Vv;({P5c@SVX=uxNLv&y@2I2J7)~9qFZk{gx&Qa6JBb3vcsg^T-{Lq)@0xel z1}r-3x1Nb)y{F(m`F@;+e?_l?N3(cT=22QtDgf76fbMngde7yN_kxo*H8D{@@E5LI zQt2ddZov{2@^7D?SG#B1=8)~{R@$Z|^FlU#^pfT7tq5U#hP6i$k4;v$0nM^fXU(_* z?Z%y(@kecxXq=Ze$g3+PE;&DuiG_p^=V(fN>u-}@(VS?K9-zOEs(cb3D;@=}WPHB@ z{%X#d!*wb6M;}3dat?jResbKK%Df9uZ&z7e{Y|XYA3nwaK?UUEV8gBPSS$D7cRq=K zktJNWL9E*>&;WPAZ&?AWu3oO%Sg$4voq6W<>5_K{$& zSC6F6_puv@_~~<62x%$EoWy)5>XYAiXw-7QFOvUM@~zj8Alnz!Df&xi>;Rq~Un+T= z*SNFPot5kheJF3ImF(E>dXMBz3D#}wRhHd%kY)E7;O5^`J9M*K?6u4_y8@IJH_34PNkQ z5$ytM8MIYHoPb-`lE8Ie&)s+Y(OHM?opcb7CFm(VM?io&vY7Z6_Sl;fP zH9Oa@4cW4KiEUXlJ2ZFULK)bEBz10Q@f~uUI$Za6}A2n(;ZLSaB<DO=ASj&QO!^zr;5GN7q7@VE+Od(M~n*4Fott$p2^WJpki6u6yq-z@n1`doL2z zNEK4-B4yLvax7c1;}YjZNnG+$oiFv-@k@;4b6%d~IEfv{`5fDcYh2_mOOz_W6SsZpS`nOfGE4bF0cUakQevvy)$RdoS8Xi&Y3eanm|WAPMYt5r^$9% z`{~OzSf{@vhpp+zNu$915JNyf%O60FeC%?O#EF8- zo@W*T;X=oh)CLet0?i$mI-KP;GWuKY+B@Dcf^>BH@+>%F3>|4tCg7vuWF3;IQ6P$^ zgD!$xstI`%>lo4IE9U|Y2alVBowTf(Z+<8+^PUAZ2R~ujEL$R1URf0=@x188qtk`_ z^=g1j`lkrFVdPntRS6v7tVZ}PRMa4(wM(6ou-D-zdUitI-IcyWAWjNkhY#8Ruu26v zt6=`fkxd80=$?G`Dqax?M#ki0zyDerutuu(ovb5k3l!YVfs+Xucyqk5zCLmB)QR}n z6Ndtac5MqBs$Ca8xPF}mx}lNdBZ11vcLi8|VS_{=X7l3FE`e|U8t*B zx7l9r(dfSb?~n83_1+0B|7W%o$h5I#3%9@VyTYrthTU79I@CoCHx<$janZZ}!cW2N z;=%IP(Xk^&jKG=|Dk^Sl48skI+aeJgl#oXITgXAKjPKeiw}Q^8y`76?jgU|`VXwKe zqN0z&R^EnCQ-|-9z$Kl6(o3U2)-b^6BoIBu)ZaWTqtqYrP(Cu#OV~@B7jtf|zZ|H4 zb#LG>Y;Y2hr!TlaFnh_$z+@ckCe6In1bkKXXta3d;W6)Pd>}5PD~^pA?Lka5`GWL< z-MKs|90uqCY^+t;-OE!XRwI-!w~J>9dx#Id$#1&z--4b~Ez`XeK%O0ZMbxv=!n@-r zWgJ?So4UMj+$z_RD%OK@Lp!Lun<%Ekt|Hx zh6nj1NHd9f@Rh*O360f!_zc&lHg4YhPd#YqB(<^_ahKeG{{l93zlV1c>c2`o{;J1o z#3N)Sc{}($(F;7m^uqvCz%_&c&bD9KIh@_y0kg*FesJ&nJMB}y9y{mjfaNkj`4z}{ zyqq<8qv$|m;fR_f)DpgCI_rBF6}-0}0$oB{d{=`D6NJFh-uV|OoK9|&b|EkPTlgN~ zn?~;)21ESLK`>1QPQpJVwD#66ijp5mt^z2VUWw%+m}+G-f$22Vw-^>c!o7{T`Tc&xRt zG1^#P7ruP%bn}%9XG@p^Z##GNV9D+ue=TvgDNu}$MQ|Djja{%X5Gk)@pjONZisl5H zyfB_sGqoI)<6s1LJn8df{y9oK%4z87$VkR9=+;!3Hhgbp?bt0e#b2ZEEH)}#-*-Jd zS?6a}iLL!+0P|NHYimEXGIK56!BP*1dmiv_0@qgZmwU$_0sQwlKx>Bw)28uUqO^Oe zX)If~a6Y?D>Euk!slOja@d4OCo5v4MM556JFKyp`p%-{*p#8^zpnxiHxQA@Y*jGN+ z8x1?-V81bc1$OV=9rP>(4u){J0S|Q?4m21C^BBW0`CsT@Z$K-BkJ$9POWV_3x{cXB z>6Df~9rPLuXeU3mV?R6stP23t#tvG0rzL)q$?H+Bdx@J{O}z_D#r&j&1+m9!pZTIl znF;wgrd2JIt3Q1tPa z@lfmbWU?gUNUjnBq;L^s+hv`!v_f9!e6X{URY0y&N@U@eFfUcCLj@2P~%xX0H zLrm|#;-yQaNrU4*4gv)__x(XY!OyZ21zerDyFN=pb|JKQ(V{&F^gBG0UXsJQuQ%RJ zOactPf;P@~IPej;)Gx5{ZRO`VF(EM{BpzyVmm+`ErE4)-lWZP6Ek@LpI5KQk$pjKr$NGUAS}%9Z`j0cKlpGzg$_s zkVpAvfSiisP`2kZEts-snWQU{?lxp3UbH>l&Qim8duwa>>cw;QO^pr3tN|}Mv48J1 zywIyo?%5tcy>m^l?A%#Xv8yAR-7=su3I2)vCu%;0}R9bkP?I$$K5<04k z!VIRR)AqlB+4>K=Lh+%89xAzb@#1*w<+swOrqG|X6jdie)H zBkg>y%M9#)4Zb7Gpq<0cz~hgA_xUWQ2D-?~f0auLbl0$QnTHQ62NYO|=Re-1Ep)J7 zqPPB>4koDj(^DOqV-CVtT80?uUKq@iFq$n0aGl;{8}g*gh?orx)ub?&AyI2@FOcRv z!1zsX0HOSP%F{q}@kg*VFiFSRVDUnh1HE_Ox}0>bw1q7WR~#9M>AoDxfQANu z1t}-kk^&zDy$2!65Bu%j*(?3&5G2>HuI7)gPD7ck32Um}VKUAA%Fr##A|@%Uoz zq_Z3HvCMk@V?y0KZ~PJHU5lG~ODj7-VtPM8(%xXEzAv6X|2`VbJOug`kbjiH$-60s za!+kwt}3&-vUx#-;Y6o-BMa4N}3x z#X36PnKV(6l0gjKa7_xyq}tX!({stV3x?N>fgw1$q*mH9l1sA^wtS8QI9ls31sac? z3>+f|6+{krBPUe^M$fu0Fn0Q_f$=kD2RLCaP+o~fKW1ED#MtqHlG0LyeZ+))X+%mnST^{b99^1XMyTb+WnkYdN zj$!T4MHNSjS{fUfrfzI*Z)*!i+uO_QE?(MC=h1e)I_nR)}ABz`Jmsu1ZY=lWS0 zxpvbzLw$3INRW2^-V-jC7OO>wa{e9~>Aye3wyCR<$CIX~b1)S@fVp}a#J-m{IDvMp zbH8=+%ZkG7XBqK8K5oZ3rY z9&x6QlBSX8OE~oCEcL-EL_gX5i)< zm8UkYC;^vI;5UtLw~%)w?={~orl&IYv7JiJkQ$zZI7G&ysY(ck<$Raa!fpPxwH2+t zDbP}PA#jDwa4wKSU&U=j0OoxIqwk#)sGf$fpD+ba`w<*UQ^D4@)qx5${EAW42z@kr zGv!BsT@ol}Px}bF=#_BEAHecQ+CmTT@CF-_zTP{s=oo~z2&e*Zd3R)TQflfq&moaGOqFI{MExN^CrwWX=3t+l0sIqmqFo$FgK z9XlW7AnFoEhG85h651&QMLRfU#)Lrmn8`N1p`8GW5ZuBWtv%uGfsUq~oIDqf`ew*> zEWtqjvRbCyfJ$;;n4WnOza%X^&C&_}Oxsq{IW8XG#O`a$fSu?;$O@+LNl3soH#gtM zw8C8LZ(1XcKnI@>i(=GacS1cqzokU&b8vXBSh1qIy}5Z_Syk0bT1O5*Lm>mesd%h) zzc&$$&IkXiRQd;@_d7&ed>_8Lam$tu1IzKFOa==XWbH@EmIA6QKNK(pT!%E~(Ac;E zO+n`u%W*9!k*?#ePK2xoX_q-U!TglpPr@kI_;1QXQMjSV=oKAeIZU82B`0Ut`)95=#S;|FTr zh9$%+D7Hbfu~~i%01tuNUa-50v~1^D(!G>B>)-6;#)&*nLhPGnEaC38EfuKSLb>)Q zaAfi}gx4xJ$R=Hv!S!2xTz21mQ{u(N%Rsb>4s;H9Zv@u=!$r5~x<4sTnfmE3??6j` zyg&2Q4i{jI1>^)w?z_80y8D-+4JeuAVIF-{-tTpEnh%2G!+K9!{19ic|7U;j6dXVF zj{@B`;{Jj4Q|F2T666|DFIlwc&(Hur!=B2LA(8;D?j&1G6IPxuh>cv|=i_1S+H6N_ z+U<}VI-)6^W_Cx2vl-6;&jhEWE(or{#2B zRyWyQ&wSFzK-r4w?#`p3AvvqdJj@vg6eFLoFxWsu0tP(8C7SnVQ!sD9U;_l~^=m_NqYtsNG8_ws3u!AL|$#woy>Mx5DsVl|^eq zI3^O6d>hZD<3|^9zgxnc_p)|q(5M1;SLinSUxdNcV+h#ydh}2mPL2$_CXl`hkA&v3 z%ZiJaljl_OI|n|}y#E3`by}oafckWLu#ILI5U;_T{B?YAczKnS9yOz9!&T>@vnf}| zI|N2BGliFm%jPrl0d^0a{2CsVzr}_#55t$55b(=z zitHCG;#`L({w&ogsr+jGUX1Ufz#hTtt{5Mm2OvlwTQ({3P!zr3j|orR_2~W9ct!8+ zPBfRyV07_^WERKWyNJ^5D+y%br^`r3W|m*_oqq7i%Ru*}a5seYa|j2~n{xjn4dyvo zPBlM7LDAky%#Y-q-?Se)rkVI^M{L0#oW=c(T(6~l>69`ayPey%sVvfR?0aGn$BDk; z`lk~05=@vX4%V)vn62h8xV^y3c_8-*1{HEn?cBD7H^pOc%#t$(-apr2EC+9#yYD6I z4xg=c>0I@xQ>V7ETvog!n_%gcVCjMd_fv+u>DLXs{H>Bm`Cn%#5 zlyHRz-`f~TA0^Hnt!?G5;OgEOUh-9kzt{6$y!eTtaQJP=^R7(&#^{t1j=tdi?EnBk z07*naRQWu>zQ@4g6oOdma=UfB14l71ISRG2C9W|R(rQ{jQwcTz0DtL?nliW(c%Ojj zj;6edt3kyO3F+i@B+muv`GT7Z+-`uOI#Zh!0o1N^^*6+54`HU4YH1&zV!7A|1iCqS za@QUTS`(fkIBiiLfpnJZf}Zx;iqDAp$*mysZra!iA=QO&HC&;%TZ8@gXw84g%Tyn+ z2tEHv#YMxtn|8mDd@l3y9-cqJRn|qm$+`6FT6&-+?NFAUv&-98(bO6rD{dCj*XMXb z{UfC%C2wU)D1AQNMo4+X#0Pu>1%?n~yaD%N0H;7$zx_hm@zJA41{?#Ri5K%x;B|23)X9^# zojiSd|Ah+|in$bLFoTsDesX&HNjAaR5>BvvQ_2yxM;S$FRpdQ=s~{v~rvCqoeS-rBXZ;F7 zaVMs_BOXOn*Sa$5V9sQ=Z1Lg`!1%t+fWk2v2ZL$dFjJPXJ>c<07}IfJ3lDv7mv#h> z+WT4qk+X8VD;Y83EGB$yR+CZCnqCfLp2L7;Hb`GUBFOyyaV4g(xpS1u2pd0AmK0EX zR9jH}ujhx<<>@?6qums#O=mD~P4q5gL1d10&IMAuRq@)5%Q#AsAENyl)X-Y|>Jr~+&$Jz*HD(Qcpes?zqrF1o~ZkwtA z<2N>M+47An%i0qVtme}zkX|Rd%;bfbky6-W9V=&%=LCQb0^MIfc%HNAyhS-(UI}v!~r^VD}$X z=3~^~6o8K5)j|a4>#0kv(FVDcHr!jANaBPQiii??bu@#I^L!C_MYO}U-li!T;m&-v z2B1?}iZzS-SQG5ASRH2g2p#74!0$C(v@>T0e-X3$YctB|#z-ost~f${BJ_AY9b5h) zOL+G&%Ay)Gp$-wPor4Rew}j25_hArmBL=y7qcT`MV}4&o8E%J$=b+tR^L|coOUtaa zyLMf1JS{g_`cQhJfExw&6vzKIKon3%g(0=S8IM?8YM(}cPiN%bfb`UAC<)MKVT}I_ zhdhkHtkPO5n3zs_(y5%9D?f`VTgN3H_RMnXaH&q0MKLr0NN7-j?($g}r_7AL zpphd|JpK*he1rU1*&Qh5Lp$Hiyw3NFtc2cxAm5G2t;x&R;o|TSBk1XJ@dGDMp1eF| z$`m;f$wOJ|;~xYsO({t0Q@m_~^2|0(I(((qV8g-b53J5`a|KKINgy~JZRr8^BKW%W zUdNv6N2Ylyxu9^zxbHww}($yr3v{IOsQZ?(PN0?p3M z=M6kO1I!rheh+Qe}(b==3&_ihC@ z;pN~kUX^cWaK_Y$6Q4SH=FBPeTTPDa z_c%>$asK+L)2F}Yps51Oe7!(M38$JZ;N=v_uR(ucIZ6nTTl?d{aFpMvx;HsGz%6v zhub+)wRY3it*gCdXpZ@)17+QbU{KaB*q!E4@X)dv32QM?GY9hfb1;|3xNCiT5u8lx z-nBWpJl9`lJ?g6M+qa+Qo5O=g!y_Af(&L6;)@REqD}OQKHDkt%Rp3|)eqprq3I+=Q%DnLN;3mA%+e;L} zsv-P_fN*G%;PfR3W(d>tIW{_a4rI%r%P;6#|4E*Lr#~9BmQX$|V|xwLaPQU!`e>mj zV=9A{A3q%pf{t>{f=~SxFU#=(`jYTPsP~mdO74dYDNrblB13eCbu=~jgq+~$xF6K~ z9qT(w*g)hbbn-ui`CUR-jwF&6*|LB+p)=A;5auV?)^RP`@ClDP#svC7sSp}K42PDH zFv_n0r%J*AMqEsua;p0@4Ch5A(5rctqmh=ym62&3JzFNvgI?bK#lr(*w}j^6wr$(i zI%Voqo$>r(@|8R_hL|3)i)S%)vn)uyNoo+JWf{Hsrh6tqa3BNTdTgbOq4-8>JWo(9%ayk7(S zTJS$?d9(Aa@>c!w-EdH*T?FAFON)?|6QqQ0Pw^cHjd&dF$+X`MwAmJL;M^AWBaj0d z?V3vtY0}p31-_PM_6nc0MjOiV66HQnyLIbf((4^X>r)7!LB`Rz&rg{)?KC({pn%$t zW&;540Ix}oPF@DeYudy~lfF*hYe2yEMN1c=0v9@Q;zR?mE>VNGL!(#7tC)HGIkW|5 zg)(agzi&h9{ZfHx=_-fS2Ns88IPgsX*=e-9&-3z6r%jm>UAun$7Mb1)z@bmL4%^@h z^a7|_&l@zhc;Ui7fFFO#P)a(L{V4}D{3RXyE;{;n?%LT+Teg!k1u%&ryz_@%{+c zT`(NAf5qz#2A?b#O`(H!;d>kb^l#L`3#6X`>{}e8cabx^$0UG(r5#>=!1YruIT%&| zb3D&RUSRcxo3b-zPnt%q=lT8Ll=DZxdWjEv!By50sYtp`K$JZQ`i-=qUBs7ASAH6l zD0I%x(MPo2L_E+7kIG`up>*w7dMd$bcTq{nf6}H~85xN18VFWdo{7ZE++XFo2YfV$ z>`kZ(PkDu}XT-2wp5r`s#+_Fghg3N+A2;28`|XGD8yU@1!AS6~1xS~cSivV zNY!-$dK{iQY0^w+GX|vpjU|PDGi}nO-P>P&`FH{1+?V<)*fhRCALOc$<|*ejOlj|g zL3{&d89+Nf1jG9!_KkjQ`jjbeLU2r^b4Y{Hx^8LZ=bBquUflNb%jsIVW5|6jk>@Z!u-el_$qKB2xGm!6)<-LL(igzu1td4q(q+Be-8tHiu*ZW9R$=%JZm6wH+R8v zuzBn<*m`WdjQ5i0Q?!rgfg#69l}k&w${2J-xJtY$!ut{&A>@sAiZq(b9t|ySflfAx z;Cp&G^ehXFEJD((zX-k#a1iIQ+B?9d066R{B^dHq%UxLzw$g>U$K?4PwK^|Lye;YhO)&VW` zquF7ZU|)t&)zx1X-zc!TX)6QeNSr!z<^biGi!fImeV2AvGh^n==drTY`^(w-(mXOt z;|ui6=#-aOI!BqgyH`5ruT6u=;YMh%kbLgq*)cLc^%}(Eq-|+H@_!J-+ckXwF48vj zp2uJ<0P!{q-P4VbwM>X3p(Yc%reOYs#;~7vqbZ~2$N6M~4)fqb@`zTNP`8VCMWiXC zJo0~7$M31+HxpPQZfCk`y|=XND$R-WESkzWQX8Zwt{co5US0eVj@mkP3@5zRu||0v zf+)0l_3CRf(V}t6;ZX+%O`t%RX>03%F$+O9#lweBHwbmri|{%ZE-roo$4Oa-6c5z; z7d+=Jl}pZE#~D=Xd>*YdX(G${HZOhBmpW7R{~4|?E$V^$V>Q%x%U=?qtf9Bk@!6LZD3F! zi-)E!1S5jJDs5VMNIRQ?iFGFTE(aW>G0OoAQwNg#n0OV`Q;ZB0UxSWf${awLw}SXp z?%J^D3I(kvejDl3>}2v*=3BTH@w=LIX5rBLw4ec!Ca!1k?qzU%l4p5m-wU(Wc>x-r zbVb875JDw!*VJj#zKSq!aT+ZQz5y-w5N25rNlXK)HfX(;xYL0>P51zW?Cf&zH1LrJ zy$)gd34KO~a4bHi{`lm>;~>-29Y0z4@#<%O&Gdmd3dBz)^_5U0hsz2bO|n zRDpNkt$L6Qr^AV5c8_q6u_WzcDvkGAROBn{a5~>qo&$`E(ZMCh!o> zVZ4%eLbp#KELR~Ibnbf@(m-3x#*s*+6jWyVdgywVOLNN#sq!a(+Fz9^%%CO?tS|E{ zD~TO+!#xaa9RX{?nQsy}y#@h9U~0+V4-k&OfWFfF+IiMVb6PEM0jtMp(HLB-5LDk} z`bE2#31=&Kr%ViQD&YX>Tfs>Jw+KO9!o7~)k}fZ>I^e09GiN@-Dv0N4Loc#z;F@W( zX00Y;9iFp^c$yb(AV4yE^f3c&D65wKuvf4~pNnD_${U<4nBiHT`R@T9j} znXpZ;!u%ft&%c9ir+GIT8j4mjT}s##YefLx*C61YmT*Jk*+4wu-xWRMXotwF;E##1 zoj4;%EA6Nhpxa@pJNPJiDRh-S`Q`fG7=t%C=WRnWeV017)dRo`gO}#i*Rky9De!uR z-_m}Mlk^gowl-C;flcz!{whxsYCIg5`b;^c*#y_ua_LpY!q(<&q{SYmqXL0PP$gDY-G&B)-;dPoQla<@zMq?*WdiBcgeifs{#n z6N%qJn=N&7z(iIW;iV;98XT+wb|bXWfhqsQ31{1=3km;poP)Dn%pc=6)J2gS>1BLoMTCE4AR9@dhhv3 z28Wlxvt5G~PSU&Rd0gwe@N)6e@2PLhKeuC6K1zNXF=_X-2f9eZD}y7atV(>{{|<+b zi&VfJq|yBKTBOW023<0dOYjQ5u2j$uz9FANW08)>DCf2r_-!1#?j3G)X+Lw9M}UX5639hAoR!hPit%yjV@rJagj2W^iK9*~C)p zA)2me0f%Ci2*$wSN1n;OL%I8bhRT_^?Y7%apy5uZ?8o?BO@6o0W}EO#9K~QKK)$PK z)5=D87FtNuP+#ALcDGKLa1Gypdr@9C)7GkSaMk8MT0N61dKFsP2OJ;QF$O&2n=M0W z<~sUNBWi^C_hHn3hhX?9bg;&(^OEE3?XQ={L8ZJh3zO*DwQD5+R7Z-F7Xix1zDU{{ zX0U}Yir;B%ZT$p}Zr|ni7;u?_AWR-P849ggzjNn+33HWO<~r_3z0ju-0(vwWNU(89 zFvyI10LQYwgrRQWyOtGks!LKU2lo)#?(T|;idE871%RMqVQo-=z_5W-@^$L3F>m0A zJAR8ejojs3rnBWwmsM5C(Jp{Sthr+ydnw`7)4nYFn;j8n(WY(Nc2WoHq)7u?>hot1Fl%W?qKC^PdGOfxFUK-BV{z^WmImMIIW^Nqbk_nQ5QG0LIMFOQ_RIrR@ZZS2+u!U2q*6 zCSgK;iHv1M5O7V)sDnPlfP4zSEuKE)AQ&yAftOZRYQ4SM>(|NeRp?Nz=_hdb0|vf# z@;)w~gdF6Va}YINoxZev`vp4DF?iq{+S-1aiLEFgL($9+IiNs6Hh>(ItDrzgfXHO` zzylA=K#Tblt@q=oi(i(I2T|KJrM#r%EnWVvIQ}pO}S!!Jxi| z`S6Dp?H!HBN1p(q`c;zODRu1_S#>-IJS6YbZtE`ZrnIyAp9|o+^H}&PYm3b7!YjZ zo0fe^x;2r{CPy#cEfYfrhUnK790yiAd`S>K&5L#5SjoklMk2!dtuirt>0LbwH$59n zz8^rn{|WeU9|?792mDsLJDQuD=jovfSbCj{kfLW*RnOTG`j6K@GumB4C=%I79*wk}FOate`!vaLgdH6}p=WqZPwh}q1~XWlzO;4g znWDC~s|-S>V|}~~pG(noFp(QBc{?UIT*_03<*@l_33D9I-d_}reiF^&MMm>w67n)D zj~+exZBp|}N=lyK+hHEnF?AWY;+~fvPkRAIXnu^_Io7>DYQL8(nE;cT!Lu~v^=Qfx z6lyDZ#RvEnL=#yvm}irh|0l(?A%;4>=9=An`F9Xx+JaV|>vDe6ayqS5lUe*wHFL;f z9;6Z8UDDDtc*`;MN2c*&1dg^P{V~td47L7R+pld$c-dL<8i)b#0pbX-V0Ray2rkXD z9;Y0_HzIA5pi}4$;G?j&yM(+~d98m^{ufEJ8hjib(vxXp8VR?i@pX0DtDvvJX}4N8 z_d{R`juXo1FjWBkzRk5Cd@3%mo=O_)RX$cy#(x3lbqJdq(u1eU)COG;$+3?Jkoo~P zE31BcL2b?6$=`uzO(~oq@6U2Uc-s_6`_aqn9z@W+)g{d1T?M!POr9?wSf6xtB9v&0 z?^WIishswcnq3+3hhPTvGiT2H zX_|>$j6+x6f`pRiX+H-fU&~(7k9e6eTOQLcYGp91^`5C(?Ifp6s)vH5OzmX)8J2Bn z8I>5mOzd*HYJ-{m6N2_TyemgViGWxeTCc1fl0cgf0Bh4sxx%|UT53&shad(Gqh3wk zhY&brFi?4)A7@l^h=|r-MwXS`4j$TTUx3Yg-vLPShw`;!LV@>RSUhx`tp);$e~~iO zdT9i+JN!(@YU%+&sUvJ#JwcWpKZ*rIR)S{AEIRc&6t}mp1HW_7TexYgnF?gAHmxx3 z(&*8X^&lZ4$EWi0@~8NC2AWl3J^4HXnH`=`?*M}&VOy;b48D2UqD2n~27SS3tLL;J z8rpl>;V+=yHC{-debKWZgvrlfQ*UolTn!HLXm%HB!YL^25`4Z7ovx5~soDZ{^WXyu z7Tn5rXG&K5Tm{Q_(~cbn$oClK-$h+~U6rA66ZaZrL_PFoF;cX_4Ayq3Pgz~$HzX^d z-l-^l=qClb>O}nn=_mInpaz=I+&E`}S7Q48`Ocj?8zo33%*6zonws7Kvp-4Wey)>= zJ+DL*!}GKv*i-zs5sn|jk;=3?wBo67j$6^vHiGjJ5K%eAOa>%C7io0?w!wWAM)fWj zzcDBYbC~BvK7A70*3fWAdI+^6K+0fp4O;TV5pS=0??WCyo*=-FrU-TO? z1T`mri65)D7;D+`G->PGm72OU;f)&q089JScXVh?cYO$ySfUmPJPy8FNizaGrQ1fC z!`Lh#?FpV9X$!9Q-Sf(Sj;RKrk`$21!H1sPu_4Ip4*0%@xgK3KPYjeDlwZKQgINw; z#wvn@h{~er2u$|Rkm>|drkI4EWlsGdbWokD9$kMUEouJ0gSf)eJVNbG3U5aOi1rggnZ(l zX(R}Bk=FDS*Gtrc2>0ftix$0$tP@yHY;c=Gv_Kyy$EC)`#*o(!x@%_zihW_C)g3b` zOl*f1R@OjA*Vo53Y}yf#8@%nvp0!<8n44#NIu;CbF%z6RigI_qGs+Tx@|U1n!TcJ+{h3fA@*9+h zV0D~)y#AM$ah%FDIMDjx@AIV`4O;@Mm=VD)nW%a0cEE8Rl0BugJz#u8%o~{6P=)^l zO$%@%F9~!7l|A{*erEoshIAftg#-z$fmUMsd;JjGt(oJMT_V zPx9LO47go_9*UO|>cC-~3rP}*_u14+66X$i5#I$B(JVLRRaGzWtZ9btVFB1q9qKf| z76u0zK)h24=FFMX-QYlMPS%t7|11UM0X;1RKl;K%fJ-}6E6TUs+oGKGA#}QWn zkeqUcnVE!IW2`MvwX@vi>9^jRKCp?exaS_7_VyYHBjTFIo&=HltNqYX%?*YBV(wk@ z?uy5TS)o(ww``HgSV2N6qu}V$*wV6^In%>(2qX_X9LECys_d@T|ByI*SUdbe++gsK z*%OlU=r_<)L=>GkI@D={dS!xF>8Xw9#E{cojH)ip0^_T_RL5r&T8nn!Z zS;jqepkY?jK`o9kI|=i;^XI3w#^d|7gaeG602)Z-B+QHPoEckMQL)C$IPu_v4~|2k zRZpKaOAJb3KykxrA!nY%(#4Da1R?%b1ZEVowuJe=vDEADAfT9KIowQ|Pccjkov)}? zD+2hfAbiE_G6_im6%3{ebfsH`|G-5AQHg%Q9CvxlCJ4>UzS~w-wjAa?5n-q8QM>9t zvM3;Ha1vva7UET!sa@OVhC&WkwTEm&aM!L~S4jA{ga;v2R>6l(jw$v9b_$o4mKIA7 zeGg*RmXPd{t`LwSCL(2JWk#zMfFxc9LqZ)QYQvgWk#_&#Atrc9`a@z*5dS*DycA(A z4ZbJ*Rc3Pz1Fn|Xodbt2@(wUKPKH8vviXa%p71)aAr8((609KnUGPw$bQGOdSeS!e zU0q#Q?IRaR#cQAgmZOW9Iravk(f#EAWdSDdzhGW^v@BiD+LDV5eHcT70{LoiL!UkQwz9Mx=lmuwX>C92yu>90vd1@(m?%a}C_^4zK#N?1 zP$)$}79r41d)wRUyAwcSm=B?uO6qi#pLw+dOQ!$FWI+ILrPX}=4$n(IU7!heCP5}% z>#OAd6Hj)Vi5KAZ9=mhOoSg~44C0(5PE(iCHCg@`CiNiu7IYTq03Wy%BtZWT9#!chW8?NjY)l&u@jSaZM&m{?*(m@L%?<04lsrz6-^Ri=Bp^9zE-4b|*Ca$U{>FS0yM3 zDV+6?d}C|h_1N7qDHFVEOW^t<5gQTe-$m;`47wUHcEA@w{2|`S!A|cj3;pgU7(UwA zhEd*G+QusIl^I^)ZJ4}oOFs4r$BWjt8-2)54Z{*4k8NAR1|A}o%uGN=8b{@G<%-Sn#b4gch_>9OV!6k)Q!QO!9g%+)x)LKht3a7A@C{@Zf}=x?k=QoW{cR(2;TG1=c~|9 z)&!+fULkKS_3IHr_whUKO~3HiZBRNR0f!dVLTADKZ|d|UTNhVRq5nfE8NBA3za=;_ z)$-tvmat)z;e_K*Xe2niPB^#$lAiMr939x=A}Z%vwc{^nu*V?s|K=e-A7p`3Am0tX zz>7a9=o!Ntl$@_;DrRVNfGoXF8g%{WE3$m>O0fEabHXi!0?4zN5Kwpx1&v6G zTG?M*`Sv%sUG(riZXyF6z|tj1GHal8@pwc_>$?vcAl6WJ>hux>dWFcceEfiVR-&Xf z&I$nNS`59+T&@op1eG_a<(F7&wn3A_y!bYPO-Vn5Ft60$RdMtpU%{<>*ic*h5_#+r zz3d7E-l#mOd-;6ilh29{8W`8Hxz8*3o9w~+|LaU+yq7ipuND1e1rsiQ82%Kw@rFX&HztPs#b{eD})Cq4epy@E^J0)CUj%7@= z%9H&?kd#2|mtbT{!5BPnr${&>gwG+E1Y1l`bLXeLjuKrF&!y~7nR@ z5X7nj!?&TKVHfxu;re&*o5NI&!D6{+PU$23(u4YvezWVD25|dDGYu9g-|O_A_O5n& z10nS-hr!W-ZWG!8T@ONsuYij-n;8j?ExzeJhtw`mP>wMbD0)6xd8B|Cz(9wf+wcO- z;j?63K4a$0xo)?pW5rO81e;>xGaET*Z-3Br_r8CQt3?2i9 z^=N~wl%pJNtW=2QI*<|MvFy@?k%YKjUPiKq!p+UnfHMtB7hIj(9c%~m(urjkMKeB_ zmd^J#W2KoYfY3iZU%l!s#A^`h8Uz4u4>s=&0DA`bf2IVhR;|k0@-Yt~+c{48Q;z2( z%ssk2$l%B<5MI1|-_$lkS{})&mHqs921Jf<8rxqfW6$9+Q<~-*S08>JDBuh9JP_3f z*);{w5NzHYZ{|mtYrF+d_9afhY8e@GvP%%G*Kln51sb~?^~8K61a##h7&DW`0zfU3 z`!|Hi2=dfYxl^(dc$tVX%Is_L5zaE5F7&*y&6|tg$p4_WFTIHH_!025JS~X1Q=5xy zM?Em=QXpqJ*YgR!!*mbC8~ zKSJws;QsQcnfP%OdH_MOVET+1zv7`Ek`{vyUkL+hK|ro{d!6$=r@~MgxAH{b>V&yk zG?}e%x9Q*`4bJAyxu+N}4A2BS-yg=@F0D)P#1MrS^4y|+Pji;MkXct7$Q;7dev0RO z1>5OQz{!2R=Ph1NnR3slId?C;5R}n*z+J&Y&_G!k3x(uxl}#|(ks!flG6yJwj%q!F ziCA@`Gu34Rd6xisgS}LJ=v^Pz$K)X6pmT2F!Jqe_#{OSaS7eV%FZAUc9BTaV1)rq{kC5phAft-2lU_szP-%< zpX>Au1hk{xi^`H^Y;VB9tl`nwuZzcaxY+P11at^Ab37HEQMgrqhQTDW;;%RG$AE{8hJA?u_s?fbdg^9NAcoDWEVX~8+_v4{6&EE`&0R?Xcp zX%j3iZolsAa_0$J2h#*PFwgR!C53JM(D)3doDd<@V+@Q=Bm87`*RiVQ6voW%SZ}iB zQG%-_TunFtY)4aG!}S``p${6rl4Tl{ZxYvFaFY^gu*IMGltDfYY7eRQrSEwTmY?Vg zQfH{IXSg0@5M7>UI^?-nKKPy}kZot_36}rOr$Dz9z@t1ebLPx-GJQ%jmjjWfg^hvn zKB?a%&h$vpZu9BogKegRaNa1v8;(ZD!Eo!r>sYtxLZNpf(DhlGs2Fkr%~*{3X&B@# z1_wGVOk2dAO*@O-gq0HIBX|rs;2q1Wse-ZKD1q(-hP(GEd$$q*LD*UArpy4V>rp2j|Zp#~QFn>{R!henSGaMQIgz4bmuui!y1` zPI2Mzku;6P31Zsj1)FYCBt|KK`32 z40*hlGT;)}6$s&(+!Hl5HQDkhgDW6xVS9l3*~(n@Z3wZVnmlUC&WVx%>Tc*$Pg%eq zFbW|$R}X^_93A2iRp;U!)__R^Y4roqV-Rriv&A^Un0ek1P~vZa(h4PYER~2*&8jBat_tjnB{qC@?~8#U_C+A^Lr^ zo%6gbVLrYHA$Pz#4p%VpJ}5~q2N~9)5tcHkU+M%(`u9FOrahr#Bw6t_IbG~~AQZw(2BiczRCikmrIb+=WmJ-D7fD+8< z@0S3F;|ceemWUzScOz(?=dpr!xH5<95k|W@!_W6`r$xh{w;(kVAXW{7PL-jr`J?4u zXYk1lcMKGC&@w$epJ6bgo$3O`t*s5Z2Wo2U*2h1#hFih~dVd@3yp3GU^1%**ZxhUv z&p`GWsNMo>Cwz>7*%>S{Ma~hE*VoILJ=`u+xH^%=0pPo>?Lae3DYKOy-(`BpdA?mD zllBsUZtX(H0Y`(uMh0S+(9)-*Me!ep2?czCK1?cds1$>PPD(ky_4Hs0b4RO4OUpU$ zImKjUhLmtwK7amX?CftsfV=3nBOJfK%L|MP%$vvKVVL$S#MfN1V3+`L2?pYhls!Z~ zXK^SS&pm}~fAfB5_UsbM?=*CQqo`(`IKjOYt+TDQxYz_b0;6B$c7pKo<;!cxbB$Ku z$?Oh6mp$;n1LGwK`)D2(UgB|yNPGKH4u^?xc7fCO`-QUz(nT;I4ZWv**lGvv`Z;s{ z{P|<`kgI7C!OAjN8)xu&%%elhTfrd&Q|&)Xn11vEqku2a3#>whIXc00>(*V9z@(gF zA~_R4+0soyc+KwJNewei5T87G(&nby!{LV!EMsBFVt8>08MyQ_ApHsgtrYR5Wop_g z_CU6I=P15O3fl}P7(5EhF}Z;2%=XQdl@jJvo19P--Uj_7(L+|Zc>z$sRcHaA;-hdAP#~Rg zq(2u*(L~H|qSEr(!;#3#9^$42dd~xX!|c8voGP88 zTz?R70*P}|%YlOdmn~Ru4+U8+O%Y~{vc=y-;%8`DCwN~bfd+#1d{TW;M|kPr(VKvQVfDz?IUh#xKt)-D@%1z#$2+BPtxTh5(;7(pc z13m}+PI=|f2K3T+BlX7#;l(n%kcB0KL7kvhq#=cL)3QCnKhO z+%${8NWkSZFz~R!op;3TaSymQBpT!S0XU(W(|of4X8CKngu1jNjI&7VM=fv)_yWDa zE67K0U??E%PgFM?ZxO5$zCBm;D8u}h@{#gBIFtl^wlnU&~bB49n z_24QqdA|Zai&kJX%kh*{OY)?~nqL7Q-3wVjVZ8`%y zMzDr_k)pWO2mQ<|ffsr!F!IIw63&4StBL5PAj?lV=d?SGIA=67=-L0V{57~4F=B+( zZT2Ai6op8Ed^PGqEUS-`Pk~-3aOL8~2eGlwkH%ss>LHz=R*OK<8tF=&b%MwRcHY|O z0UC5+E5R?Xtfa!uYL;*$~hJ~NQG4yag4U<2MH(%`FbtQ%2i$-9NpsHQMI!3 zuadc%I?$ZQQhj8=)t`BEfp9@(>NkxXuuk{umWEJhzxT}qI@1~bo)YPlSATyd-tv~W zwC2M1qQ8jv>2pH?U!doPjy{y3O#!VDHsK7He~&!c(Mq+S3}QY0J@zKmTy6-hF~K?0 z(i|lWQg-M;EV>CL30I`-$zpbP%M~sVV}gV!g=Fcn74XXQ(v2c}P-?&xyFEF4MtnNZJF{p1Rc)~IV)V6Nc# za0ZtGx#}E)TM9pSp91}-K!Qz2v{v|?S_6$RD@K6*96^2nZ{_uqm$JTA%e%i|_ zZvr8UqND~EZ2fwUWmzTwvq;}LYVej@ZZQ}zhKc$}2DUpMv^c`gO9=O(Kyk600rL_( zEnIW-TghNAgDB-QE85&VD=!)50qB7P$(4)<^HU58xiM{#JX_vPi4;QH$wSF~kb_MD zU!V_m4fy#FIt7^Xw0T4{^S5zL5JE8G^9Yll@*#pm3*!pc(#GKy9gw~!+7Djxot1FpxJ zs?noUUgP~s6dc}(M+$fDajtMoCPaP z@H}tjnA(2h$9#1q0%!_!P?0tFdj_C7aHO35ZzZJqYuXpwZUaKwG7g5rTkv6MWLsOQ zwRff*=DJN;7$4kLxT`oSn1X2E`$pQ5tQyYCy)Y24B$qnnRhVUid*`^E^3^`RuCCt>PRw6V zga+F7mg3v^;8hM}Y0%&XbzNp;g?d03RJx5xd%U%cV)YhzQS;F(9`xMBC0=_`ch)jN@;Ya~#=MvQ3&SCE|;S~&i zKNyQ`;dug=4njZ2eC{EYzou}Ru?Y#L4qA_wXc>Lqg4@R(GxsC}(I^RY3yH*bBs;GD ziRNH=c|PaQ(-9AP4FJ60a0$r??@}$;;@cS^g%ZB{fhSt`j z2(;-i;V65SA06cV+WS%)eV!617kYIyL~TH z0K#Pi&(2X#I23wNJ%l!H*|Msrt?dxFM`&#Bbh-hfU!fDp$1k@O@CABq>FGloA{1cF zxm3{jvit9!4R?K{wYBwPX?giSmPaCUkrqLACu=AxD|=;#6ekyGUKcXK;H)%&FGGts zgB+bD?$)0x?m&ejXy_Mhlbb|hg69~gyay78zLwPn9`VxBC?@wdd>m{a{2_vSU(4*L z8$=5D0(}sP?`J(|6bQO~!C6Z52**f9kPd&2nY4j5QZq4)MiF59?e_p`qk~3`>lvE> zf)+ESoy!ppTI`9jI~*G3H3ZT?&7Y?*8Z%~$Hu5=T0zDWU$KKo;xL^5lvI3#KEyszLK4$9Ep~(7kCGuiGUK zK|UQ1_cGSww^8`r5UL1sH#^IQw4Qn(L*ntIpMxCYjv>eyUnDBP&6(VXOx7 z`VASp2ePvLymCVUU!doPjy{y3Ljlu}7B5~r3zU~5CH_LFw-*)NsT+dJ_G_<6YKJ!s z^L17W1~qqT$(n>YyTu(tn3ge@eud>~&Y>&s=62ObdLYBIW-(x=I_^%OD~eea0L?%$ zztZCt@bgf@wSdX!qt+7ze1YB*b^LF`jsi{-WqtE+A+%m(So2!h9X33PBNWxx?(g+g&b z(ynKuZkoYi&t&kw=92=pu|_^K@L~HDC?pEtY!%beCnUvdIC10;gd67NThK-mk>cVV zg@o%s;(|#yxgkw3cpn?WW1k3ZL^Ces*g4L8X?FsBaD-AkevbNKm=(lykSGQp9YCW! z1SIeW2d9Co96v9g0vS?ZKu6pe;^as3DUcTmxHZgG<>kMOutGWC^2$RGJyhcDK(@~_ zE8mPlLS+uv7PyJTd>%IiqtSh=ovucZp7&0fQ`eaqs&RDqw% z5Tk(KF@{)$`RMkI0?fM>$=i78f(7@pwt5@S#{~&tJ~JARYZDo_#(F@t(7u%>x2e1n z=XB=ze0jXZ6Nx8+MMYz=T(p+7wrXRSK@FTB*K)bKx|xcJYgEBDz1rn3Ab$v^PzDvZ zJZ2u$KugYwTsGix77Gn@u3ftpyycc#;+jIrBMv_3Aw~gTpbxPM^U>`S1%eXhO3BV} zqwwN>PR|JWu?TeafmikWW(`hI;B>YK8f?K@@k&!e!v~9xCdUGeBL-~%0ZSd7jU zU$SUXGs3)y!B{-f)@A~IIMvt6l`D(NPM!=Opim=6j*Mp_tZCCCfyOukUN_3v4~0ho zU!WIWo%k5#i2}=)FR#L^yl=yX4Yma*=H;nqrAIwsFD*z4>P;6Oe)!>-Cma?a9lj~; zuP7L-LU@|QfFOe+H0D0`SRzgTZpI<5k$L^$D7>^cf}9dlC*^tdA-{&v?2b2zwmKdU z|4Fphu~<+>vA=!^wD`e>ty^EMsi_HZ-IQ(jrt7S|6xbK&y%5O%JS-?+LSXXb$xWCj z8&o1RqSD63#^M<>W*pkQd9z*(M?s--^%yYgh7kratQ8|6KndRMKVm1DnM|lRv1{C* zn%iqSV&201$T^nKg|u~D$4^84R|5J^!g4elE6Hq1b34~8$~z?%jZR{uIga|Mz(FyL z4?#fY3HH?ww6(V0&*Qs1VSdvNNTGorJPO#{j86@p0yhH%(&rly=Hr-)J?*_yKNxO8 zBBLtZ?wtvboLoqPovgz@i4|i8!sEi)wQHTZJtJ6oj-)-^4@o-34;&}GmPooX`nZ~xe|df8)opN*Ca+DJWhJg1u@v6(i$0L96)L%N~2M8 z1Y{SD`oZ*W@7$n9j~=c4+xMfTp5nq6EbavQkgmJKhYuV5 z+rr^7_%4j_$}NH2Q&5>L4*mp36p;a{1pbhwh05WF{!_pg=>4am4`GN=z&jdsh+&<3 zbe)-2TdAU*+wq=k!8XS2-HyO72tk1mFGhD=4+WaQGj_)vcWBRmAs^gc_0e#+nnBbE zPa^jy(21jHMQn^D`}|=ML8}smJTd$N%`T37>J=cO=#+H>l> zel-cVYSk)Ei%ISfhlg0bm!lJgc|{gn4ZslYCe1?}#?-Jb;=)}Jiip6Z;yZnJ;Cfe*EQnS~rZ7+7kqD<4h*qN7!WrdzENXSi1^qQj|Vc1i^7wE&jiv0rm z6zG)#5+uR8y1Hwaw{00)uPFvLf(bh2nd{5T%iVN9=E84aam^^V8t9$o3otRGM9D}= z4L%sO`Ez!dD+tHMEb#ZE=8*zg32*Pb`T|5Rdz99!Tp0?i zSrcEfWXT9r^a*I`nE+8FZ9*fv{5~b{9pW+}+szZl*U)dCy7o)zQ^2P{fl$C5i5hNe z(~`BUG;|4c4$d(ReR^wauXFANQe+>E0ib{{&&hhy9cB{L8SL6 z&?WNa;m-YHSy`D0bRB&=z^MD#6)**SfnLCsrN z$BFKM%HSO92D?0d{)IyUU!WIGUHF*z6!0l<6HtH+VIlPN9)+p{(UpM-OriwRQNe@b3I>F8J3Z z=i^k4!3e<`aNgYw&Vxkf9_ctpIAv2eJ>uULAg>*(4Gcy*NcA_-7E^MZ{@0&A6;dB? z_2kOyetUv!Fv{srNx%16SDo<|eQ=7?9`3gmg~I`kg_Vc9ws7q!u$N3wyTlz^yK9%* z{BOh|OrQZj+&!nT0vu8ENVG(GU;XQly zgf*|k@sDbh88Z|*SP_jd!~h_gO}bnVE*)=~&}d|R=}=;#;cM5f1+^6<0~Ik0!O2@^ zax|MWXHG0roY{A}STv)4MCY92=5X@LuRMAU!bvn6jJi)pE=TPkq`ke}cA#F}Ltdb#G2|N>)ylbZ zD_g3n#xn0TlF@BhFcu5PLm{TD63x839En6OY}l}&IgP^BQ3rrg1Yuq#+`8%vVpz@1 z&E?=c8eB(!UorOxyH5m}Kx^ZWibhVxx>8Y5ah1-Z^{8D1W4hz(>O#j$N)oveSfaNw z>rA8N%a>OrT3W_q7aS1_g-V4hDoAr%B5{c|x98LG?2Qq0RhXXMb=L;&z4zV<4)YiX z!>s1m%@QUcL)3pOw7d-c&QXS@G`PlWyQ+`0IJti8QHaq-BGuK^6|ttKF|nee3bg@D zYQ?mH1ZQEla&~zG<+uzps_zbOM{8{ns$IqU$oPzTmYqBq_S%vJ=?%hq-n@CEii?Y9 zB6!rVP7x;)p&pz)dv=6<96pMf9MvnkJI=zgs``la`o=44&xRXOUd%-MV& zvI1R1?>zeBpo2MV)UX7`H=oM9i-t9mt(8VWka8|{7&oeIgL5=c_zS$-%e65O3azFw z@8mg|E`!;Z%2k!SVr*=C4XM*1t?h@wZJQsnhn;sEUXyIKW?{t*p zNv>n0Kg40t`$Ey^zLKh{lQh-_-sdvd0KqX(bzp?#ieMtKf=uRv&us9i=2^6s(C24B z9bTRK4BvNAwwLHs&#d3N^$_nIo%_>SIt;+Tsk|r{Tuz<8hRo-Yr)W8vtAa}!P>^@+ zd=qWX@%#$U>#6%^sQW#9n=XVU5E!7{g-vxn4Co%pa2vQy1Ci0( zD*;@_C9|yB0o9#|5~tZK%SGay;O$YE>H*Y?{lwYd*w}a~JrCaXCAhiW4%*_t4kw29JBdVLH!h%g=7At$mX+69ewf{OLa@EPeUbQ$S9Z0v@m#`Jb!GL$T?6iV z)em*Ml`^bpkH;UUA2lJ+_tJ;%WKgETjM`^UAxhjL6p@c?+`RcKnFvWQj_QrME}Rp+)l-mA2I2{#P5O<^l7oy)*>KEUUa=WGB^r99&K>NDc`ss~-7 zgX*t|?+wIP|3A*}qbMN<8SKA`&_DRlrcI}7^4s^xGJk=(7;?epYut!$>C&Y$e;>|rKBDm?(SLf=Vc3f&D(4Gc`4A@XN8ZrQTV66d)7J(#BHrfUYv z7A<-$oaPa}ujIOuHc;BR{VJ`>VR!zh3>MUec@S+CQZ$YsBZI*!BsxN>gVfbdnAUb+ z?A^S1^AU@j1d{~nz3Ad5z!rykpF~~XPYYcH+y%Ux!?)?+TIBFm`_pqY7Q0G4-v=W< zp5AuS;CG+{rss#i_xFJLI`XJ+<#2T;1gwOVld@PjvAUe4&OS(et?Jyj((>u|V`tb5 zzTZsw-^cYj5SgVpaD^1EY4xP{u6|S|U0xsEOM<8Q{F}TRMPG;$FWLE(=*A!tB%o~w z04q1t*1piwK*>ukcoJd(PtjQc0xQUO0k38QDk5;5@n+ER+V!=yFLqUL>F<^_7Bt7Um@QVPQok0(4nn(17Fp_JE57BiP^Ln#_>u1 zBABjB4t}cr@Ivtv((j57|JTd&BtFdj0QZ+cc1NJ7=vB|!WXbp=a0MHZr7KwUIw01e zVizu4csGsdk0Jc8P@&Rft+P-$BDl0zM|hRdT|8RQH+Sh9Z7&G8kSbPE;JwUK{h)Ja zPA1LnuLnaN%kWvUaG?a@@A6yB%SPp{Bk<1Q-4pIYz~QdDt3!1bWNnghxCSe70AHr% z^F;VHn8Eri%9Wxy41FY=`)`2r+u&X1!R4pnY>b0vqorP^1y^=y$6*K!nT_d-iTxd# z++TZfmDZIzE%^PN*y$t%T?}oH~5RAq| z3PELKsL9uETLst39|*(%EUjHZ<#lD(?^Jn-AF=#{!M9*t zS)C475&IiFyi@rhz~0A?KcUVR3)fCRudP5ZsK zV9vkWvvBp)Htic0kJm!mPlK!!22J6)I@4YF7{38w{Xjttm!eUBfI;7PI@g=CP;i$O z?B7G-@S_wW5SJhf1l)9|@>Tg2^dZi)Nh$OGf3&f-_VZcRQHMT5zGLA(9|7j0ltTmG z^s=ZAC_k0S1=nsIKJI<0@AJZa6W@eXN#5xqdiB#pq^B=lx$>(zUbr(|T--vrCsmn3 z!KNELT<4Nz^WgmX{gvk7Y7ao{wUL?RV!?d=bf z_y;oSX9J{DTr&q_NiZ`Vg8WT%fKSp8m#g8R!F0fz^6=J7s(sBg%OBEEGx4~W)(M?nQt7l*1Ul1ej51YD1%7rN zy!I9c_-o*2OPD{0Ov=)Dr0&93m(`mGC@Q{07^^?&LG`Aowg=|VpAid(e@30%OM4K% zb>--4x2SMyPUrWOr4+ONM`?V2M*AH{IlAgIqN8hr-GO!QyJ}m; zm-NSrBti&gl(t{a)XGQ+bA6*-xH$S*a3RwMHE4*dF+oJ}rZ^+c_ae++rec2|iZfjt zjj9rORt0EqqRQ$^nXCf&6H$ZVZu6T3`a533V^|D?r>7JUnbuo~XA97e>TmR8;g zZ}Jr0!lSQ2-BB<^1JObpHAL%6@^#FuWVuhkGw4``hv*LD{R6bU)eH*$|Mt!V%(ALF z)OD-6s=FBk6%i4dLBIi2kOs6xgNZ>Dhd2b|6g3(R8Z~O3`OJ%^6XSahag1Y3Oo&rr zG|s_URG^`021Sj^)C~LsfO%_y6ncef!p}d(YXosfPA@y}z&OoO{mRYp=cb zn)cemIS_KQrX-j?>PYoPUf~ub^Je^*N(}$QImZ!KEjN@8k3DcY{`luIYCIQuJ_wxf zQ`#z*3T|c0n_zWU)xil`P1-diN0}ytcNkbvDF1^8?1#3-+>tJGrr*bl`Dd)&VbsH5 zC~U5EWSV&09{$Z$dd++gJiu>ZMyk4Q$rZmALK$y_#6W!DZ{75X zGUS_Xp#?3UlSWc`a1Iwbd>%#l4vmy_NO}6f%f;thsdORqbD`LMF)(j11G3g>>zo3G z55&`{sVB0y3TN}(bjAL@<^FmMStVz}pqi3#{nNR$<*yg?|Z+&AUemZ_Be`tZvV9jXTzD3Ac!qSK|D|n~;XyF#o-Z)cg`s;6eYKYk zLsGhRWd^>pzcuFc*AYZ%G+wWjN0nO&DYIE_2g2B)FOp%TjxwH!u})yr)NdWMX|HWG zwX^=U-&lC|4IG;TlI|_h1I(t3qDtHA;P=%6T<*V`llv~ zq|%bGP{z@4R_BkhGIV~&ZE*^N4zYe|i6$>op`zV0D7EK8^LL=kET#b(2KS7o*M4cM z3l^aDb-@3g8R+XgikOohUWjn~3CA9LGVaO0sAT!mnD-0y^_K6(CExev@%L6N-Y7hF z3s&h{{W6q3lXo4JFUN=)oxu@(7oD&m@qu*Ih&82*YljDTAP;O0A6SOzS0W-V_#vf; zh;{T$ZgcVl8Qk)zunD6n1w8U^Bz$(Onb)pe8zWw#Yx`}ToYa^vO>~e3cUJ$jJY|jg z4q#&XvDSC-#TS>*)~vHq);BIv#0sq#1J6kY4vFR^yr1DogF$_k^9*%`_7$DdM_DLr z-vq5|0j0GWwRw78Ld)o9oUptlaFl-gJpFbk{Z_KQE+B%O1|3SwT;gtoRLww~voYto z4>I#Tbn$E9*EaI#;BBHqgC2Z)ehS~eLG&M3e-9jg>YTPBqMCGPc5?gV-Dw`(S%KOi z4Y6}oh^)@MFC*L^;we84W>SHhN|d<0Dr0ISSmicb8FjszPWLiEj4oiDe?0TdGh^`s zWqDkdG?a#e%}q4nql6USssiOtyq%L(2nVJvoUBt`8rA6KKW-gi)xzilTv#y@CT14~ zz0S+bLdAl^2cpzICjKO8*rX(Fw}0Z-{m0LE9{s?52pU(NCynw4G2|N>oQ#-%mKX5Y z&yWOo>mlR_e=2Pf|R%^+mVVeBVSKvyOOZ=Ru2=3V#YpLyJIXBnM= zcUkDmHnWNkn&4&JDkkXS0eQjXizXmF??XA!pYFx|h3}e_6Z7oB`{2D-53Jgsfm*x0 ze$&K6g!Mh=fxf#3istJ~H#pkTxown(+Df*)bP-^Lqq#oH$Lv84rrnmM`OXMJOpCDE zp2FXT#P`a^3qqIZNNJUI2_%>+yS%s0GEX)mwJnFR86-wD>k1zuFQxrIj3bh3jL1r! za$#WdUi$4sN9MGvNcRe;LpOE~`I|w4bhIsVlJs}*w(Qtr9{`V^hs^vI^fxNv+9(Ha zbxD{Vx(4y-z^pgD+8^X8rCF{k+k+c%X4d#H(`Yy*g8Sp3`!^uMy;URzJWjI%WS*Po!h!EbMcEEm6(CwRpYqnn<{VH zc}fw9ky(u*A62M?gAlx%zuhS3?u=u}n57K69_DHr;l{DG6@2c+N|>*|tq z_nF@a&25M8$N9Y%fRq^{umeENeGFU?xcb&xZ@tmCo`?ut=4RZ9o=D`KCZ$nP%!5sB z+43cnKsM$Y#iq`+9Wh5uIZtVFB=CBgN|2l;3@3&4*f05trH*7PS z)|lVK2=K+OW>hif(RLGY^gOdcVrt+cDDf8pONT;y^L$iaFff|j<$)1_FL?csl~S(< zk7c|#hWm%It?Oa@9mW08JYULk;OB$#wX{$83gX~I#zeH+7Hr+RW^hIptZ?k6?^%WY zB+5J~6jKBKWM|Q99CqP6*?Vw9o`F(aCjQ(%9-6-dQv3t3RywK{Wi09_#_oH>pP%l0 zW!>mpey|_dv+v&77Xb_1);LNq)VF}9-^%#}9|4ckpyi{P{#XWXzYR`*2%L9=(>dhX zs3MS-JX^&)_o-)bsD|}AVv`=FDf2GZMJV$BPR6`cF=-S=;9F0%+Ja*H68-&B@IC>Y zk9p#$ryd!6uxizjgn=Kyvtyz68!56;J#K?PJ6R4hyDj*X}=9!_J@Y@Nr}D>oRk?Fx1MsJM7E!U?m3Py zy>t(mS#t2fM~zQRELC@s=iTCxQVC7Wx>g?H+R^i={7!iZCBBnjW@uH9YUG;Lt3N_V zJtirEQsLrkkfc{cqt&$8U6J=)T7@bhPf@&oH|G z8%1(|lt^B6+?{pSy$Y6m*KuTKc$q9xnj=^g@Em)Y)}h=>QBWQT;V7k!_SMN%TeqHz zN;|FV@P%3Rryng=c^?DpBRNdx1{B>lecm0}=vOGOd}o8~pI{)xN!r%&tj63?(9f8l zeS?b->`C1y#6#dCyoi>^P*PFoh%{XBH3jaj7+6)-7oUhYo zJoq6a(T~7uA4h&qginp)H8~%M4Y_7!Zdkv5{kjaa@Vd8Up-y(}`xZ;@BYHr8-C`!J z`S=zAcolfX}ZXIW3}WJkJ0JT!T*-ag{13^9S9CVBJ?jbqr>qmVyD^yI4u=rvmxE}Y=m-(XZ;z%?78f@(t$7DD%4 zr@^{w(}szO57B228NvY*Ue-6~(Yoik=E8>>E*BH_$gVpyM&U!$+SoXU;9Q3De+D6m z6X_3$lj$OefeE6mPR>~4E|t!2nS8G!6}le6 zy=cX%Rp0T2`r>vLMD~%H7J6WzYdL7-V~uuzyiSMs4obE!rewMP(oEg_-dp^Q@OtSaRtw8yBPhRpF{XZ~qw}uj zCYTmRUSl5Ho1-AzWwHWbpXu>TJnxP1lwJd;3#fY0J$B!HDSi21`cfV#*HIe`U^S;aV)XPktgKTo zM$3vv=J5VFqsgx=Ir!kGsb7Sr4uNLTH{HAsHf~|naF0OOxH8&)B-1-Ss_Qr3n;Un13krMicu7~zNVS@e%=Uja8by<$7Lq3aodSs@J zgH_4Q$jHIsAT_x(*_N1sKzD{B_TxsgB~eH_Un&=t9f&)f5nO@C`c69d9z%G`xq~m> zyk*Ntj$D$kPSdHm69-`)d&SdB1*_uBLAiULXQ9D5e+-=Wgek4BHZ(;AdWLa4D+2HW zjd?OWdfk&fj5w>t&Z=W9Gc8Z~O*+g4tfgVH=I;UnFPloM9HNIB)Hj?D`e z>20;kV74U$7d*X0D zkK!&&l;=II(01_{L(p!&ilcrz#@zP!hmxOWHE}3j>Re5Jlk_cWANlSe^581^+LI8o zvof`4(O=SM2ZFP*Yc!`R8Bx)oTmwRXu$(rjZ&JTCmDO!etwz4`z28hKJA|~h#c+5f?baRd#O%~eMkQ8axr+~@y)2a*P-`Z zuzLd2M2{fS@^CERZ=;LPU<&0z9D`R`7j5qCzs^n?7~=iZ<9innH`yT_VCUsGL5F8C zZ4t&i>3Z=Tn9?>~KTzYB;0{Uj7?|G|(eP?;@M~}kndvMNtu1-?zicApj>fQ+1%gg} zlZuB_+d(OV3L=yuO!x{$y4MUNT*BDiq~|Lm$si!=>Uj|^p<|9F`~dIuZ@TJlY_G~d zg=ZOP^q^F>u^gEl4NWtRvBomaFQu}5FEG2G9khYFgDgVYz9M#w%P4Y;P|4-r)x0gwt?+)7!oe+ z_p#RFONpN}=8dUKIqgp;9a^ZI9Bl~u*Gylm^`=QVx%R<))Nu%-A@7=KZyr-r1aByx zmhra^x*z@+VdVZMc>e>5PqK4Dmf1<(N)-mY*>4B@YPD|3x&#NIFTPnv;DjCll$T+g z1=d&xz--UB^oGg))M-8YKi4Oarj!r70fwW*{VNjH*Dc-bJIZ*f-c{DoG+m;uFduUt z_u+C6?@3K>ewBl|s! z`-^VhxN({E;oY46g=P-k90g{Vu*Gau5@sWYMp;%biNFRf#%J|5rR91+rkt4 z97^$Xl+iJ$!YVrq+W_3k9zKHo7t9elM^)lsI&VG!18+e=zD`IfTvUW}R^+ZRK)N4H zyGvS`7Zu$&GCm%PJg*fPSU72Z+EmN%sk{qQTmbXpIHr+@j)4_L)wa-Rj$89MBqc_bq!8Kna=CXQy_GlaH!X^wK9_nqvJ309^B=TlK#p?@3w1E z!*&pM&aY&p`0}I_C3a?{LDg2q(X*hdg%=t}J--s>7zDHEIm{Tx(Yg>0#&~gwmz@FI zJ{tDnVLeRCXGPwgo_;t&=l;+}MII8FMrMYngu{rko<`MS3=GT~N9ckxwKVFq?X`H_ zpHQ&74CtNWt*n?(E^*?Rbw!Q!|IDNCPb0i9z^H%Ct%>5KLh)UF@v^t+OJpOl5H9Ts zfZk28?UFDaAo;iS$*thD$WeY-ts~vj5XU*ARruMUvTy%pqv5Ytm`i+up}7q2E~O8C;gjV1moVHz13z(2n3O`Q-wfR0Oa zx85eYSB_Mxb%g-CXxa;Zr~92d|1L)iBdmfQad8Ct;&+BHjtq(d?JpC)TLlQc!cwJp zXWH3ObZDtR#>4mf+j{s-G>jFbFy&3mHT}H^WUsLhnOGHWUK+kE38@F@)G{S6gDEkH zSEqW!QvvCh0v6GOP>N(|YEy<0mY=p6F?s>5=_II8y^^hC4AbZf6BRn*%2h7so;P zXMDVx&(+o%^(ReFPOkDxXGsCYyk~dj?YQ5zXMbemm5CSKrq)C=9UtfU&ts(eDeDEN z;!)mm+LhX11D7QmtKzi1b*WNLd|Sdy*%=vBK*N_5<|x4LKPX?pbNkA9bi>rII_sxD zy!LM)|#x;DbItz-a7jIrru}^ zeP+bxxWdS$pn|8M`|5shcjjlc%)bK>?-(hp-P1WV#L$pNxd|V6Ar@rOL1&1c`*pdL zGPy;zD`iD@4Bum??rsmCpS3B%b>#S$EEF|LZ4RJZufl|v`Z$#r$$w4tMn;AN;bkHW z(zbdggl$Q$I_M4&FHG4E5!I+* zLF_&LNOURY+q0_v8!{}{hj9M&O>sKVx_2{FI_L$Uzit7bQ_+y$r4n7~SP9*PpW@fMT2MKmRS?GtFBSaVT zebkpZiqJ3Ppl8{My3gKwKZHDP$1|%u+LR&>|E@G6eIlRL;+?0}==5?L&c(tyy4QMq z3mq|rQxT5(-0X!jyQsKM+!}oGbIi|dwBg_g*Eie7J8}GvR;W71IfM_tB2NG z9haT9`(~Qfl-X%sl_-RHM>ZQNyTE$1C{;<#$AEk7JIxD@NxX z8Z2C};8YkSuZ}HYjxa35ravx_Vdj6#pF_|0SiJa1+J8(^2E_&hje1prGqA679rVKw zk+oY;dbVpAy4r8fsk^Pk&u#uI2K3`7={+1-Y>!dLaf?+YB7v9F^Hc@H?X{MM%)N0F5UZ?&ESOc65F*wi0!f*thv#cdlmeo?r? zD#yG?kh#$l;lI6@c9A|!^;%UZC-zFN@kf(#%F{dap{s)9P#KC1MJH%tRBFK#yu`fg z@>Fka-{r98B0i5yz1fjys<(6}xCdZb;l}aS&N` z^jnUS__8!*nnRS|0iG+!<)MM{nBYpU$U`G9_gU_n^uF%}{K8v+h6kk#O*HHWC$IHb zJ-%?^-XZ4rkUBnf;13hJe!WQ-Ea%(=cs` zB6k~+9?%*g)`j^K!?w^_SHkpPlKDv?4BiZp4AHUnZ>tNTUd%q&3kb74GKUL+Q_OFM zeKoq5aJD+-{@uOz&@qhZ5crriaB{THSEGK96R#wpd?b`(iFpC*05 zsib^_Z7Kh0X(8WoKMZ|4{Kv@aDMRpIHh7y^W*&Udq?|IJP{$$sEux(wN;At7nL`DzC=!Qz06Y|M!mb0)GGlKV%Jg6uf?!mNSgYikNEdQ<&mTemGoOhznPiOTgXwW z*I|-npm{i_lsh$*Ezuojjpzyr;7WgZ(xKOy0=LMG?~=j`5o_St8z zH=>o~rBC+8M}aeX;2}5}k8?FWscCd>o+w8r{fj@KV^iDB)}4YDF=%YHPL&osXoNxI zv&^jHER35HKxy(DkxH0)-W!rf8&K|%2$%}jP)-Rh(IJ#OOQf<_@BI!=Q7<0;p)#c& zJ`b8k=Yf)pgv;kj{Ow{&7np=1X|Xk(6z`#@AA(00Em-haLit8%#III4K?~bGLEVOJzha$DM%6q$DflmTIkZ|& zDCIxu&dTXV_6O&4z~zm`ge7?Zv@)Nks73vag{lm(Uz_$`$C{A{Q`6q)$CC0O_rTJ6 zK`OWJ=fVy->EkGM8Pvxo8EVuusoz}CXD5GK_<6+VeQC;r7j(zblv$<|NGfs+G^*s^ z2+nIQ)Y(qxk3I>v)2*L*dv3FZjg2wVO(OzG%B2c$_T zWemg68wfpqPlSe52$?CbUA_9zz&f^+DXmMU1nFBr^y-qj^}zLJJ>Q(Nf0IV(7M1Qp zCF+&0JP5*Y_bBo14-tZYuRx%|)}(k#V1}^;x9ic|ug^Foekm#6oyR^DC(c)Cvy*T9 zt$Ayk3LnK&dglIX9CG{ z)!uid&~c0q55SldPg!;PZDiy};Pi_Guc;kf)mFFgR_HpAI1Sw~!F+JZ6ipb`M2DLe zZP{|I(3~zWc-KV?=e?Yu`cq=NjE3sk0mFCQrpv$c%-C&{bx?+f%gkZos0Q=2hbe zM5x?IXgay=JSyo14#>F5o9q~PzavvN#4zOJQ$4|%b#;=VQsgY^LNKl_E!Phw()7ne zGG(5cxRymchOAAEID=;hE+=`toe|c*vsh6hryy)^@^hdwj&cC?8ow1Upka)9QV1<} zG@JoW@5c~}lS(2KrSOJKMi8)#gy7xUwsh%NDIfE54Z<5?Z=D|U{}zsc)iybG<_YHt z<$5-^_QB9HCk@06C*qG%y^Zo>y`rN*GQ ze(N#%<=*S(*>^k`w*PN9`_fC-3%E(&b%o>)KwsoN=)E}A{f@OBgvGDkxMjg|Pnl@Chz zx3}Qk`FmLAnh#tE!LjU3_Q!au-Z_g7{Kt zBXdwF+DYDXQ^i?n72zQhYxrW%J3Kwa+>`JtgN`E429akawCaS!7w^9N3y3Uy(@_Ow zopUTHPqx_!B8*B44Cpit!V4Lxzt0>=ITi8W@NOF3QnKmcQ_hcqTl@8os!Y@az(B z)iG!c`&yR9R}KZCUw>1-uxjh?3dtXmM)$S(h1-+z$oOB=u?`jh054xjL_t*5vEVPC z4GW_K65b6yWv7ZCSlUD9o|jGis_u$~#LPDY?VI?vQ(?Nv^|I~eMaZk`T^s;3$!Z2O zmiX^VALQxb{Bmc%mNs(U+d({fKRRx&M7$C_@~AUTOah^jGItv9B*=bw=z-v>P|A@_ zOsNnAWAJsf2(xBoS&C5JK7@q5nvvu}l(J0ORBwQ%@^F60e6%*?U$N3OT2KenRC97W!djn0tD zsg)KKO8h?)A7sXUC{5^6DL;CZ1xl3@Mo?#a*cI+E#Y7+LWrc&36l){CT{JnwdQtqM zD~$&xa#RLyWaK7z>S{Cg8^LxN`G+K3)6{3uS_J&Ot7nd0A?z+3{aKP?U%h-wWbx0` z8>HGc^#G0bfovF-tPgtf%=(!Z7P|f+y|B{8_C{X5F#vD5)?*5W{fy7h0fBV9-;RG7 z9~yF8a`Z~zg;POw*KFRpHQHXvx1uP$l^Jx*J)rx`q&uBiN_YD`WCj{uB|Lc8XTLG2 zw4w9XplsOa*(F^bb0gD1)k=W*IDp8|ad!#sS&`Gr4;@00>wJu144p3v#u$cxFj0K0 z!|#*R?w2HGpod0Y`eq!H=v&ZRp932Bhd1>r{s(%s##G)peMG)w<^KEyLG z<^#;kam*dpeO?FS<04pnMZGo%P;OSw@1u9yq@yvx4iQ?v zSG2THe*yD9#H;fIRb^w>2_$N(IV|)IyvooiM`;mq*V;FPii%J_tz4FxsjVv$U*9U) z01;rgDR;woxw_H6YELaLHz$j7kH@p!s@TBKIOGXQ@cnp7HIO21M0#Vz4{+O7x2NvC zhyq3H=FOUm$V=^U{_Ycfz5K4^DwV*5(qF%i3-U)Ea?fZB# z%`R^pQGnWwwy`xuA#P<{L}A?oN~-!@zJ-c00!>q4j<)u8y%@a(Y3YT|B=X&$h)>#$ zsig9O-8TiST-@%Exz_;^AB=bY7QKoA{SM`+5rG&vgLu1L4X*h!-)&xnYkn&DQOm-` zeKKYaMXTFpDqOqTbz}Ixr=Lb1lsqGjT1Uao!6RXS zQh>sSU}+pFSAxpvyf21XC2KalH@-6r8gcRYExwNO638+6eF@oGWIy%ZXw}viQ2pYW zdXRyZlqE>vb}$VhQ}PNzpl%-gNG--!5Mb}%MPOd}?={3RoJM2-5A-l|{>r6Q1QKm9 zvh!UC=B0YtJM@XRpGI7q2Q4CF{&LDDTzU5mp9V)-RFPvMw`&v-no8|v24=cqdTBG; z+O;(H*EX=W@z?uUbrtLJEuH1MgYY62fXVP9Om?v_?Kam=a^p_YkKfV0?d9;DA$C-E#Ct zcdoJYJao`46W^Go$L~N1AJ^M8J4?7-SwMMZL^>wjAXk8U>e^~`-ZG?hh5f0@Gx1uu zU8A=ql+gY^y~k~W1q>}RV!kY%zY5~y~0Vj|y*!`BoO?}j!PpZZv>_Bd0Y(gxYDB9dCK zefv|TqWwu`JCo}$f`+l&zmHz!7Pn-9{Rsg_K?mf;G58B?l6o7iD^gM>z}%M#N246LTY0;8$>0%Y0=0h0r;b9)KQpy#e0>b0F%x(= z20W@PlVK76Fm4<{zZ>Osb0*)^49A%yK0JfFoPn6eQ5DexihFj$KAbC_MXn^!Y3jRc zg<*91j=+)qkJ*}ibS%WOx3`gV!?4>fd;(J0u&L}1y6zSb3L0!#YJ10Jvaj*oheY?he0?4H#spDh|6xV zE~;FVOcl}~()|HD>PSBlWP*CpnRj+`s%cR)q8I8DGQU#(`sdKmv;61ur>X%n^PWc6 zJG;8s#&2_sr?1#<(}Qt4z$WPbD9wkNtjJ+@Bq+Z*y!}Lswz@vv%aFfM8U|2S}|M|ea9 z_88PfjdqA2abkwMO5GYPG-I@`2WXu#EzYM~SvZtJe|BHa-3ELvo}ipnR|+F#zEmJ1 zKfKQFh^KKtgrKb^bIKPyU)mrCtb2h^<)3B9zisv8SQ)Aq$Cny^z%0n?JsKB0GSl>7 zVa!|kh5gW4_crwau1y+MD&3(jin$`TQd*}6>;B^LPZES#xRfjV_bCd zBY|gn>!jFr^A?L72ViH^H|#srhiTvIIc}_&=~B{B;w!{cw5(l*wyyp1m6R(GccZi`#aTw=s{sSwGNAwhjvj1iWT(^_0u;eA)55Nk)QY15V z@P)@6G*lThdmRI|u9cU`QW8z41;Vb!R$=6FBwI^q({%ee@f7IM(>FtI{Wwwrp8cNA zQD5;dLeMJbLb^Lyw?Cxr7EaGRD5=4vA{JYi<|Be;0)ZVFZ#h$~s z>{`Y>J0+WZwt5pKfuwZy22qRP)R?2Kj7q}E0Z+Ai*sb#Q(|Xu|0B+S>w3kme_QFfI z_hU~TbG!L@(ccbNofnp5GIY1Y@f zSm!+>cvR1VHyW(QEPCt)e#wg^+9H9@(9~^8n!GCK!~&A6gjDH(-6w*vC-y#LQw~Zd zIjwWufxZ}&&WT#ggwc-0Ig2F#pLs{3rCzA}3)ca}sEGEDllM8dfut%v&)TJGLPf`dY|ng@^UAnoYu(mGtf@jw{wXZ z=iA7o2))tgBE#J0B?0fPsDIjEkgY$hk7(R~Oe<50*6Phe?|11%bLq>>Q;}DzJ@%Q` zTc(@GNR^+scT&rJoQxmM8I^XmJuc(O>}XnzOsTXNBMQF?%}38SRHnYQp;z6l zNcHJuz3ce0mqb%5^b8bg*mYBq)kaqjPdzC&m^UoMRD0pb5>K&|X^*Z{k6_y&+3UWu zr{!Y>O+FqThU;xY8WoxzQ|FD?J01RYD)Z-t$5hobSOtbi`FJmjnLM1-3{{{bwWOY1B$vhI9!Y zQ>}Nr>n+6sPV30uP!9%NDJsykpsJh)sct+_DVg255;ZypM#8wYDdxIa*(l%l+xQ}v zEdNekjU2C+=v~+SaIi4M0vw3_DnuB~`Nnpo$$KUktte=Xl$6KZ!ZVe^SnIi8Nd<3) zkJ_yG5C6O#>B9Vjt}lm{$2%GvT7RV`B>#?uyi^y1Z7cI|EwUyc%Z(ydVXB|ysG;zn z1~nsdsN|y20S~)qB{kZRc)gW$$Fh%(mcc4Z-Kl*SfUBlaDjcOVP{O@XVUayoalWH# z#GU6;?QfZG#{W$T>j1u7%E|rw)b@#qDIAU1~Lwc!3s34 z?L0m8!O#*%ZP!$zJ}%)1-u;{!1M&;Um7xj592(Ey$hbL_SKRvyAbFZ|&IxXC#%l%9^C|3`SO+S4i$mvuV4)W&E5|CX2lFMe=S{-4y@EuE?Koylp2CxC2Xm zI4m%=ZlG+82sb#K)eL04NemsTW-sPp~rMBS1#sQ1FA2-St4 z5JM`ZRm&GwQBD3?3^Agi78{emtf(~{)vQAe zG~G;0Q}VL{>BWqA$q3sv9(@q3_&;=z5Th#U52&1wmCvtJC1U!>&$=0LBT*?*BJV)X z1L@dE?dGyQxRdieM?r#vqJO$*V=aKGf*k<`=Eu-O$s!zc#{Nc(PVI>B3T9 z^2=m-%|U&9t9j}p|DFscTc)|}jrn9o%V?DBr6gn~wC9hfbI&(e7$LV@f z-H*)B`#$gdO72mdAA)W~S8V=66o5-f`q_t&+yoWd@C$v4_kNJ&FrTKu)U(=%$H#M` zMw~g6g5_7sQ*{)stN3~or^kOeqEB+Gw%{S(E41c&F*tjExH0g2sQ3#z^6-p-L^xza zccu8EE|=N2^o1*71NY0BFExnWL1Ad#e@!uE zmp(!f=fl5SW3j;7LYwGer@2NKU#Lg*j{!8kfri2^aFoczWCN305c^#_rIR8aVp z$jBAKU)Au8P4w~m)}x?!+RX|!!{M9-hdB%3Os0k8suLWy^GjXaMf=&=FJ9^ zUp%szIffTaY6-XXF=rZmxew^0EoM2p)?DiLD|C7?7`(z~k^+;mbTuIp9+uXJvpW|I zn@DG41#M4Elk1J+xSaQ;V?TdE{bf?ZA$BpE3Mr1$F6*KI3WNROSIUgm7+2j7wC{qs z^9Od04Kij~?-YuRlJsJK(!Yy;P;~fKV;rj95k(AUlQeEdT#X`=j$+kT1<{R94GPh? zu+(KECPN}V7DEOrE-tuQ?IP<z#Ny-Go<_XhcVM5k?!jl#ZBJv71tk( z1y`Kj+w48YniRC2_mYdQk%n#l)PX!b9~ZsNW|LSv_%g|Lg}8NnQIL-msXRZSy+{y; zYaIoaIc=>t#aTo5Gr~#<3MWjiu6~OaeF;diU&C)nAcdK*sa}IoL5+^SyaAUcd=FQ} zgoJF%z#s#3)z6>(X{c|8rpIwA2OB+V4@F(g!0&J-?#0_Z(^Rp%B8tru9LeCLoAG3| z23w_OSV;MvdGa;!FNmk#B{rP9aLY2cj!mMuBYaTuNqlKWZMXxY?no=QM6Quspmmim z&P}hR?N^^(WGiA&WrTZdn$%RkV(%g$mNHD=DyrzzTJa=&LC*Njo6L*V9EKyedV1ws z(~Gr>vYUG)B*)e`+bwhG0IK+%TFvF@DT8{R8}UQ>|9+wAXtQO}(nPhr-**L4zNn&! zKj@sEB(B#(A#fME!>0%CX{a#d!d6&0P}Ua}b^R2&N@u1WB>R|7&jKBuLmpAtO6BeH zsJzd3tYNaUp_KdgK6lI1YSGq9UE^206q18!Z;XyjS@}}I$APJ zn5!<-7CMCT8xd4$(rs!xGpcl)3Q9o0P1bOnr|#>QE9X_S_fAxLok-H1h|(|Feq9aD zqfZnRp11@9P(day!tHA5R;MrEQ?l!RVEHvswYQR8EKURO;rz}tB6M4-rl2@jrRETu zqeKR#Hbm8v*I&%_`(WBzU{}w~vnl&Tp4ULvLsF6IDpSlAPjjT@T-wT?yi4s?kF{ub z?#p-x6cz@BeRR#%c=r>nCehZm^?i-z$|`~j3QUgd3dY0ZLpplk&=?v8_RUPkkyG86 zm!*`p71Js!WJ-D6-`Qs}rJKNik#|X@z)4BOc&AVwX7Kju6NQb<*o5Gw zXyee*k~o%0Lcg02x04@<(xlRpE;u8~mUQ+^!0lgfon#Trb125XDSyx7V25cNdAE!6 zS|s;2M_f$Umoe50Im&)ucxh>BxY7G!dmzDFfKUhVX2xG;7*#rg^!x-@E<)hw^X+Vg zU_Aky#pGq5xL4|woE1JRCCWh~;%{5B^b%dh3`wLX1xM5;SHN{HmDT(thKL<3AEOy~ z)85cOLEYhPc11SC*KWtIz z+|`;sRa(7WXSILh|B7q-B<0`2dhlV+1m4U{g*2B62B%R^5fW|W-vD9hJ9#3=akF^# zrr%6L0vyAZMMXp#rTbx$~J({!HPuHJ;wsSR^TUHp&7q89=buRY16>a%A-0}F0@YK5$QVwW{xEfLk|tskwR znb)d3Eqd{RrSW5qa$8yV+~A}}r)W;ef0*QE!PKC$_=tU&+G2+$W4%vV+4gG8C%#b~ zSw{rv`EGg>QpTL$N7J>fuK{eu1tvdaaGn<-yvI6uqmV7cdgJ+5G*H^-re0j%v6XHD z>U*buyMkI;ZGQ|`U$VR6O;tvqqfyW451PI?118tm0fk7AoK=SyxRCTz)M97#SMBsG zM_J!AGS7>#cCs4;y1w#CWq|!O$hA7ya?orNw;buDZTEfCM~oqT)T88cMjxwb;`N2~ zAfi99qviRjU$@4K+NM|vf-dTZX~kh-nBoHgLo(&MurHatmCAMU$Q?XyQ8{&c+n!NU zXr{~fB^rE9r!weW#S~VeJ*5_oR&H$VM7hn+&BkvRk5I99f-&aFGyGT{MAm=$g*m9h zy@;Ne^YJ@Jd-3&nQu@T$Mxgb^^05KfukWTTO~(OSH5EQ_s3LY0`^-0M^Z6=e>dHpOCasNTA$ z@zj&B?|2Njzf4>@Xybhs6r-Ryo8I{|$AL#I^;eC@Oq5YSaI~kgS5|deJskY|KD(wZ zURp~CnT>B9Uyf&oDvW2Y9OKAD1KRW`q%xi?+2p)<_ec#_0;UE_D2N~dI z4Ru!L_nof^vc3|yBSxQQoZovzdZ%fB2~#~Kxg-+NwiyzJG4u8=WQv450uw+%k$67M zYD-trtL9dC68Yn#5|LNgsSlrO+6oYP?HH=Q`P0UZebQNBj4dG*)3YiJMvs#JaMrT; zPfgC|3%F-5Ul@doG#IVX$`&$$1R}2db~pfl!j9z`m$BcbJwTBy$ zg>D`ueH=TuwIsucwg20(RUWC+>s>-bo4n(*sn-sl>OcXP44#EGiK;lLR%qd-wR^x?5Q4P*b7L81KFb6ZEj1?g|7(1h|Uxi7el5YPlzbHwT43xg&+C)E%pnQ_qmV4qvvW5Sf3;)nN~Zv zR2d(*AQu{ifrzf^vxJw{PDQY7Njkg-^kQ4DNY|*tE`9hqr(0OpArXBAe$AQtJxxE# zD3-4wF{!5w{3TzuKCiF7K;+G{rV^XYnN%a~gaO5f7pIFAJnjkrTbbnUt0 z{!S}{Q}S?&v7!#wO8j7l(q}Pg(S^A6h~@E++p=eD%8|UCwQ*;|kC4iV-(buY9<;iD z&cRL8W&8E3*a}xJDC(kp;$WmQxJ;f&8H);~`P1yH#OWeo2Xe7l6d+14{1m*psJ<_O zt%Ck~VE9=*>2LBT-YzMBj_=+ln2JU)&$(RXU#ZbnC?quxb^3Skwrqi!)K8xm1N5oD zxXtPW+NGKit~hge4Fjyn5LN03d*ad2Un?L&Av}cp)deD13`uIEU~55Bm-sb)MKy&Kz4ACIAMogGB|yOEsCVo&-Rop_0toTwf}7}`1wSc~ZU_{Ej8rdY+-6yp z6tu6#YKfA^x;0>E4Qfee*`WYE zTY=#=CQq6MIm{bq??^$l2Hy^}`CV97P+tfEQgqiVL}4d3LRTMnz6EvOyIm^9&D0KC z|5htN&Db=c=o~tljo25`Oa~ ztHfK1|CmAXRX!RUEl-LXW1|NY?;t?Xmfa`z2-7_{5_WbV6uSo>H_M2R8Jn$4a{nU$p115|uI{!z<#M1l$=$rr3>mZf;Tw`jhDJ z-vw>Gw@CI!uRnk(U)kBiOyJ3U*o4EZ`?GcFAr{PW{5jt=0Zm8sC?^`4jf2l zU-um^06JXKedajIlq6JdGAvJ<=iUSVLnfo zc?Z)slLQbBj^HK=&(%K~;2-3p{oln=IPZ$k8;{PF-YFYqtt7CX=H!6gdG5Z+7oHI4 zL(mPgsnXat0`GRvo?oW@2>bYZRq31YN1siw&Vd8?K6UU|z>y}TPvQc73g3c%a+9Sw zAj@phja3QHAL6)?>^S}LZEj$s`#;5RDh^#R&kD+lb7~7X=B=>36zn!W&RwnuPJkR(yqiOdK?BEYjUbaS?6?;ncZ(G3=R$@ zGFrk-W=A?TI9fyT#n*Bk1c=n`M9z4PI^-o}H%4u5EOVbd6d(Foc7qU%_Vwe}5^z+U zBkUx8q)aY;j7PnEh^n6w_YbTge0#GGCZALxcMd7YB0{d4@iJo5;x2_YIoBw`x$!&K z`dB|(uwp7Lsz4mbG*6f-$umZDI|sz~haRF5Bpv7EkIuqR44f&T`qP;4riJTVJ)}#ojCjPfyznp`jUp2}l^Y2hcnP0J-iijRF<4eH#nPLH--9Z@KRid$ zWo^21IWsWq%FASZfoUX+;)SjL$4RcXm0ltgmKw(!r<8}>e#OMZMDp>=b_Q}8DRVw% z*WM2lOLKf0mDBV}=CH9jxcPFaUJP@gU7|EhC=54$a38I{V^@|wZ7i)-rL;9yIB=KI%`O5Ib_BIyaaYa8UUN50nygJa zNO68Ii-P+S%g{d=vS6INL(oQ2&NuhDiOrI3%dE|EM$RyDxr3b-T3N}KPa%Bt1E<9= z+%+WQz8m22ysn#%DKRHSV-vl(NDC3me^<4s>$2BrV0?qSY*T7*D6;omsNOOYHS6`g z!z+8lxMmPOxXgQ5Egk3%NSNHCz>rj*afz|5fMMOO3^X~$CzuuNc-_=%@kN<-NcPatQo8w)>Kv{V&1D0_jeLfto>h<$+j1Mrocmn zh)o}`!&&1ZWv>?9!kgEnez~xALwD=75Ws2I&zQOVSPp~yh4?{Sns-h)d)g( zvf|sVgLuO9g!E)+{Z>t}5;Df_A8CXFNpd>haAj-J_jk93IuL+qj$j~yPyoVHUb?R* z-hU9Jw3g>f#)BtTI3%OOd8*>gxk#TAqTKOskZ4T@hrvEEL*1BCtKUHtJ5YME*%mce}bEdl4p|qEuRLcYc znv(006SPzx{*5(WFyl1tNn->BI+|~HO>NCf2EEy?);ut|r35hC z=Goi42oJ7P%6^`2e-;SN56KVuwYhMv1tGCnm>PeI;N`(#u#dc%_*z5vp6A-mekV;- zj1&+|q5#V=913xm&cDIE8w%}25tFT(MzwC*q(bw7Pg*@aW92<^!#LmHIl!Gpe`v>7 z=be_n1|9NzR7(`ST~dG)1R=ooG_pvZ5As&F_48?yU>@gJUHKjt1K^kfzX1n{_2kDL z4J!PfL*79yCi;^^=fYo;3=ZpJLpKFoQC?Qd3A>{A*kae7S%C-Zi6a770Hr~N#Gvcq zyPS=`dr6)z7uO-Eg(pKh{;m?CTbEcD?4nnhT{nFD6fZ#uh;UYF54)qxkQ^+`WdMzv z=w@E7ONpD4?KTUHOU|8z1Pti z(7^K5NM+3OIl?t6C!g{m{`Ria+6P9mFj}A-S?o&2@iFjp;BaP$s$OEKEH?zj0K+H| zAtR8rNd)ZXn!Ri9bISCvMPYkM7`2<>di#(#m!4Dpv8SrkSHciQY%I3#j!N&5%+(*D zv}N|xPRbW1Y!Ed9mg(YWsa5W_V?;eN9QVKHN+n1rWM+=lvgkbs3Q=Lcx?@{ui|K^( zd#!~i0I|JHq{MXVK=8{mffQ10l*{i>YPbX{nQ}jAa}dT|dUq~Bz-h%Z_t-FGPO;;8 z&X>FYYm`*mhHM?_`5oI!5pBhz{O%*lH)o@-Id8h-Bl{vT2Orvo9s47Q$@Axz{Ch$j zZXTu8e3VnwtrxVH+o;AF;2K*W;ETl8^QUht#+f;2#$NlQ$MA}9T4t)wcv{em3HaS+ zMXNzbAn#NVatBxv#x^K~ezEqG)0nV(>}AG^p4U-t75CVi!n1WigZ12nsXB1SIVI(I z(%eXuR(<^xcI7uDc9fLGJomyp2gcoQ&G_fOjwa=@?MLj{g2wD?^`hDaBb*vW^5T)X zfD<)_SHJGQaJ^7Xuj4#>xjLJ!?AfElW$fe_|9b-h8YZD8xAV%~A9;gceG5aVp)BRJ zTna<&s}mBi$GwXzu33Nb;Lf@7@n!7!|E#@)mh(1v*7uIjE_z8xQ>p)TG`8U^767*e z%RhT@{&m`~U^5BXDcBCo2XE~{*#;^SP(PbQdu%F2`(z&Q|Li(pyP%N$d3r!ktioXC z!CTpR{l{Y&rxVr^k>GVz(M{!KkC9Lx+9!Oik zsk1W@SQITHcF98@-TB$oxW7;AA@X|(sPMWbV2{lr&u{P3kVS$pleltAvE6^azB8Kz z^i`}EN8@Kj$Vn7m#V7K)aVN1A-0eLMOolV9%*T_&Dw<_Sv?XvRa36US@eWZ=Uf=Q- z-(sH%eiLCfaVy0pFWuYz=P=YiLVSS)Hzitwxzixrkc^tHg}b#D5w*zS+o$3NxH2D< zR84Mi`bxg+%r~Ud_J;_TDJwjYOVwoQc4U~~LLKxnjJ)*%yf|k3jeV5iVs@%+yD+xO ztLvS|wP>w^1X6zp_M^xA^LCIxgH0-lE057x3D%vHMY@^Quz>R)SM?3+8xN`Qu-CKg zAU4TL0cGlSnS|1xCStN>^Xz@I{iFVeG=G=ad5;(Wqb%RCuiI2KS%GYpFmy)!)W-<_ z(RP4oWxq6wvzez#Iv!AxiAv1r)(f!=*hd1H_ERG?z3EZ%crBc>?;2bbSbcf~OLOr- zaaReuNX62{KtkHx+tVM%lhf6U?w(_Mc9d&41#VpeN@MraxFSVrXP+{k55!$s8F7QW tC4)C~u9tPDI519%y8oZHr&q<$SJVMV8Hk6(FDHbzr6{K=TP **Token Management**. + - Generate an **Application Token** with the required permissions for read access. + - Download the **Secure Connect Bundle** from the Astra DB Console. + - **For Cassandra EE**: + - Ensure you have a **username** and **password** with read access to the necessary keyspaces. + +2. **Permissions**: + + - The user or token must have `SELECT` permissions that allow it to: + - Access metadata in system keyspaces (e.g., `system_schema`) to retrieve information about keyspaces, tables, columns, and views. + - Perform `SELECT` operations on the data tables if data profiling is enabled. + +3. **Verify Database Access**: + - For Astra DB: Ensure the **Secure Connect Bundle** is used and configured correctly. + - For Cassandra Opensource: Ensure the **contact point** and **port** are accessible. + + +:::caution + +When enabling profiling, make sure to set a limit on the number of rows to sample. Profiling large tables without a limit may lead to excessive resource consumption and slow performance. + +::: + +:::note + +For cloud configuration with Astra DB, it is necessary to specify the Secure Connect Bundle path in the configuration. For that reason, use the CLI to ingest metadata into DataHub. + +::: diff --git a/metadata-ingestion/docs/sources/cassandra/cassandra_recipe.yml b/metadata-ingestion/docs/sources/cassandra/cassandra_recipe.yml new file mode 100644 index 00000000000000..78cec5ef4f31c4 --- /dev/null +++ b/metadata-ingestion/docs/sources/cassandra/cassandra_recipe.yml @@ -0,0 +1,30 @@ +source: + type: "cassandra" + config: + # Credentials for on prem cassandra + contact_point: "localhost" + port: 9042 + username: "admin" + password: "password" + + # Or + # Credentials Astra Cloud + #cloud_config: + # secure_connect_bundle: "Path to Secure Connect Bundle (.zip)" + # token: "Application Token" + + # Optional Allow / Deny extraction of particular keyspaces. + keyspace_pattern: + allow: [".*"] + + # Optional Allow / Deny extraction of particular tables. + table_pattern: + allow: [".*"] + + # Optional + profiling: + enabled: true + profile_table_level_only: true + +sink: + # config sinks diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 2014d8ca4e4ddc..3152d0290ec227 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -404,6 +404,13 @@ # https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/release-notes.html#rn-7-14-0 # https://github.com/elastic/elasticsearch-py/issues/1639#issuecomment-883587433 "elasticsearch": {"elasticsearch==7.13.4"}, + "cassandra": { + "cassandra-driver>=3.28.0", + # We were seeing an error like this `numpy.dtype size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject` + # with numpy 2.0. This likely indicates a mismatch between scikit-learn and numpy versions. + # https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility + "numpy<2", + }, "feast": { "feast>=0.34.0,<1", "flask-openid>=1.3.0", @@ -660,6 +667,7 @@ "qlik-sense", "sigma", "sac", + "cassandra", ] if plugin for dependency in plugins[plugin] @@ -778,6 +786,7 @@ "qlik-sense = datahub.ingestion.source.qlik_sense.qlik_sense:QlikSenseSource", "sigma = datahub.ingestion.source.sigma.sigma:SigmaSource", "sac = datahub.ingestion.source.sac.sac:SACSource", + "cassandra = datahub.ingestion.source.cassandra.cassandra:CassandraSource", ], "datahub.ingestion.transformer.plugins": [ "pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py new file mode 100644 index 00000000000000..6a5236563f48db --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py @@ -0,0 +1,476 @@ +import dataclasses +import json +import logging +from typing import Any, Dict, Iterable, List, Optional + +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataplatform_instance_urn, + make_dataset_urn_with_platform_instance, + make_schema_field_urn, +) +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.mcp_builder import ( + ContainerKey, + add_dataset_to_container, + gen_containers, +) +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import MetadataWorkUnitProcessor +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.cassandra.cassandra_api import ( + CassandraAPI, + CassandraColumn, + CassandraEntities, + CassandraKeyspace, + CassandraTable, + CassandraView, +) +from datahub.ingestion.source.cassandra.cassandra_config import CassandraSourceConfig +from datahub.ingestion.source.cassandra.cassandra_profiling import CassandraProfiler +from datahub.ingestion.source.cassandra.cassandra_utils import ( + SYSTEM_KEYSPACE_LIST, + CassandraSourceReport, + CassandraToSchemaFieldConverter, +) +from datahub.ingestion.source.common.subtypes import ( + DatasetContainerSubTypes, + DatasetSubTypes, +) +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionSourceBase, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + SchemaField, + SchemaMetadata, +) +from datahub.metadata.schema_classes import ( + DataPlatformInstanceClass, + DatasetLineageTypeClass, + DatasetPropertiesClass, + FineGrainedLineageClass, + FineGrainedLineageDownstreamTypeClass, + FineGrainedLineageUpstreamTypeClass, + OtherSchemaClass, + SubTypesClass, + UpstreamClass, + UpstreamLineageClass, + ViewPropertiesClass, +) + +logger = logging.getLogger(__name__) + +PLATFORM_NAME_IN_DATAHUB = "cassandra" + + +class KeyspaceKey(ContainerKey): + keyspace: str + + +@platform_name("Cassandra") +@config_class(CassandraSourceConfig) +@support_status(SupportStatus.INCUBATING) +@capability(SourceCapability.CONTAINERS, "Enabled by default") +@capability(SourceCapability.SCHEMA_METADATA, "Enabled by default") +@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") +@capability( + SourceCapability.DELETION_DETECTION, + "Optionally enabled via `stateful_ingestion.remove_stale_metadata`", + supported=True, +) +class CassandraSource(StatefulIngestionSourceBase): + + """ + This plugin extracts the following: + + - Metadata for tables + - Column types associated with each table column + - The keyspace each table belongs to + """ + + config: CassandraSourceConfig + report: CassandraSourceReport + platform: str + + def __init__(self, ctx: PipelineContext, config: CassandraSourceConfig): + super().__init__(config, ctx) + self.ctx = ctx + self.platform = PLATFORM_NAME_IN_DATAHUB + self.config = config + self.report = CassandraSourceReport() + self.cassandra_api = CassandraAPI(config, self.report) + self.cassandra_data = CassandraEntities() + # For profiling + self.profiler = CassandraProfiler(config, self.report, self.cassandra_api) + + @classmethod + def create(cls, config_dict, ctx): + config = CassandraSourceConfig.parse_obj(config_dict) + return cls(ctx, config) + + def get_platform(self) -> str: + return PLATFORM_NAME_IN_DATAHUB + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.config, self.ctx + ).workunit_processor, + ] + + def get_workunits_internal( + self, + ) -> Iterable[MetadataWorkUnit]: + if not self.cassandra_api.authenticate(): + return + keyspaces: List[CassandraKeyspace] = self.cassandra_api.get_keyspaces() + for keyspace in keyspaces: + keyspace_name: str = keyspace.keyspace_name + if keyspace_name in SYSTEM_KEYSPACE_LIST: + continue + + if not self.config.keyspace_pattern.allowed(keyspace_name): + self.report.report_dropped(keyspace_name) + continue + + yield from self._generate_keyspace_container(keyspace) + + try: + yield from self._extract_tables_from_keyspace(keyspace_name) + except Exception as e: + self.report.num_tables_failed += 1 + self.report.failure( + message="Failed to extract table metadata for keyspace", + context=keyspace_name, + exc=e, + ) + try: + yield from self._extract_views_from_keyspace(keyspace_name) + except Exception as e: + self.report.num_views_failed += 1 + self.report.failure( + message="Failed to extract view metadata for keyspace ", + context=keyspace_name, + exc=e, + ) + + # Profiling + if self.config.is_profiling_enabled(): + yield from self.profiler.get_workunits(self.cassandra_data) + + def _generate_keyspace_container( + self, keyspace: CassandraKeyspace + ) -> Iterable[MetadataWorkUnit]: + keyspace_container_key = self._generate_keyspace_container_key( + keyspace.keyspace_name + ) + yield from gen_containers( + container_key=keyspace_container_key, + name=keyspace.keyspace_name, + qualified_name=keyspace.keyspace_name, + extra_properties={ + "durable_writes": str(keyspace.durable_writes), + "replication": json.dumps(keyspace.replication), + }, + sub_types=[DatasetContainerSubTypes.KEYSPACE], + ) + + def _generate_keyspace_container_key(self, keyspace_name: str) -> ContainerKey: + return KeyspaceKey( + keyspace=keyspace_name, + platform=self.platform, + instance=self.config.platform_instance, + env=self.config.env, + ) + + # get all tables for a given keyspace, iterate over them to extract column metadata + def _extract_tables_from_keyspace( + self, keyspace_name: str + ) -> Iterable[MetadataWorkUnit]: + self.cassandra_data.keyspaces.append(keyspace_name) + tables: List[CassandraTable] = self.cassandra_api.get_tables(keyspace_name) + for table in tables: + # define the dataset urn for this table to be used downstream + table_name: str = table.table_name + dataset_name: str = f"{keyspace_name}.{table_name}" + + if not self.config.table_pattern.allowed(dataset_name): + self.report.report_dropped(dataset_name) + continue + + self.cassandra_data.tables.setdefault(keyspace_name, []).append(table_name) + self.report.report_entity_scanned(dataset_name, ent_type="Table") + + dataset_urn = make_dataset_urn_with_platform_instance( + platform=self.platform, + name=dataset_name, + env=self.config.env, + platform_instance=self.config.platform_instance, + ) + + # 1. Extract columns from table, then construct and emit the schemaMetadata aspect. + try: + yield from self._extract_columns_from_table( + keyspace_name, table_name, dataset_urn + ) + except Exception as e: + self.report.failure( + message="Failed to extract columns from table", + context=table_name, + exc=e, + ) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=StatusClass(removed=False), + ).as_workunit() + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypesClass( + typeNames=[ + DatasetSubTypes.TABLE, + ] + ), + ).as_workunit() + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=DatasetPropertiesClass( + name=table_name, + qualifiedName=f"{keyspace_name}.{table_name}", + description=table.comment, + customProperties={ + "bloom_filter_fp_chance": str(table.bloom_filter_fp_chance), + "caching": json.dumps(table.caching), + "compaction": json.dumps(table.compaction), + "compression": json.dumps(table.compression), + "crc_check_chance": str(table.crc_check_chance), + "dclocal_read_repair_chance": str( + table.dclocal_read_repair_chance + ), + "default_time_to_live": str(table.default_time_to_live), + "extensions": json.dumps(table.extensions), + "gc_grace_seconds": str(table.gc_grace_seconds), + "max_index_interval": str(table.max_index_interval), + "min_index_interval": str(table.min_index_interval), + "memtable_flush_period_in_ms": str( + table.memtable_flush_period_in_ms + ), + "read_repair_chance": str(table.read_repair_chance), + "speculative_retry": str(table.speculative_retry), + }, + ), + ).as_workunit() + + yield from add_dataset_to_container( + container_key=self._generate_keyspace_container_key(keyspace_name), + dataset_urn=dataset_urn, + ) + + if self.config.platform_instance: + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=DataPlatformInstanceClass( + platform=make_data_platform_urn(self.platform), + instance=make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ), + ), + ).as_workunit() + + # get all columns for a given table, iterate over them to extract column metadata + def _extract_columns_from_table( + self, keyspace_name: str, table_name: str, dataset_urn: str + ) -> Iterable[MetadataWorkUnit]: + column_infos: List[CassandraColumn] = self.cassandra_api.get_columns( + keyspace_name, table_name + ) + schema_fields: List[SchemaField] = list( + CassandraToSchemaFieldConverter.get_schema_fields(column_infos) + ) + if not schema_fields: + self.report.report_warning( + message="Table has no columns, skipping", context=table_name + ) + return + + jsonable_column_infos: List[Dict[str, Any]] = [] + for column in column_infos: + self.cassandra_data.columns.setdefault(table_name, []).append(column) + jsonable_column_infos.append(dataclasses.asdict(column)) + + schema_metadata: SchemaMetadata = SchemaMetadata( + schemaName=table_name, + platform=make_data_platform_urn(self.platform), + version=0, + hash="", + platformSchema=OtherSchemaClass( + rawSchema=json.dumps(jsonable_column_infos) + ), + fields=schema_fields, + ) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=schema_metadata, + ).as_workunit() + + def _extract_views_from_keyspace( + self, keyspace_name: str + ) -> Iterable[MetadataWorkUnit]: + + views: List[CassandraView] = self.cassandra_api.get_views(keyspace_name) + for view in views: + view_name: str = view.view_name + dataset_name: str = f"{keyspace_name}.{view_name}" + self.report.report_entity_scanned(dataset_name) + dataset_urn: str = make_dataset_urn_with_platform_instance( + platform=self.platform, + name=dataset_name, + env=self.config.env, + platform_instance=self.config.platform_instance, + ) + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=StatusClass(removed=False), + ).as_workunit() + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=SubTypesClass( + typeNames=[ + DatasetSubTypes.VIEW, + ] + ), + ).as_workunit() + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=ViewPropertiesClass( + materialized=True, + viewLogic=view.where_clause, # Use the WHERE clause as view logic + viewLanguage="CQL", # Use "CQL" as the language + ), + ).as_workunit() + + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=DatasetPropertiesClass( + name=view_name, + qualifiedName=f"{keyspace_name}.{view_name}", + description=view.comment, + customProperties={ + "bloom_filter_fp_chance": str(view.bloom_filter_fp_chance), + "caching": json.dumps(view.caching), + "compaction": json.dumps(view.compaction), + "compression": json.dumps(view.compression), + "crc_check_chance": str(view.crc_check_chance), + "include_all_columns": str(view.include_all_columns), + "dclocal_read_repair_chance": str( + view.dclocal_read_repair_chance + ), + "default_time_to_live": str(view.default_time_to_live), + "extensions": json.dumps(view.extensions), + "gc_grace_seconds": str(view.gc_grace_seconds), + "max_index_interval": str(view.max_index_interval), + "min_index_interval": str(view.min_index_interval), + "memtable_flush_period_in_ms": str( + view.memtable_flush_period_in_ms + ), + "read_repair_chance": str(view.read_repair_chance), + "speculative_retry": str(view.speculative_retry), + }, + ), + ).as_workunit() + + try: + yield from self._extract_columns_from_table( + keyspace_name, view_name, dataset_urn + ) + except Exception as e: + self.report.failure( + message="Failed to extract columns from views", + context=view_name, + exc=e, + ) + + # Construct and emit lineage off of 'base_table_name' + # NOTE: we don't need to use 'base_table_id' since table is always in same keyspace, see https://docs.datastax.com/en/cql-oss/3.3/cql/cql_reference/cqlCreateMaterializedView.html#cqlCreateMaterializedView__keyspace-name + upstream_urn: str = make_dataset_urn_with_platform_instance( + platform=self.platform, + name=f"{keyspace_name}.{view.table_name}", + env=self.config.env, + platform_instance=self.config.platform_instance, + ) + fineGrainedLineages = self.get_upstream_fields_of_field_in_datasource( + view_name, dataset_urn, upstream_urn + ) + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=UpstreamLineageClass( + upstreams=[ + UpstreamClass( + dataset=upstream_urn, + type=DatasetLineageTypeClass.VIEW, + ) + ], + fineGrainedLineages=fineGrainedLineages, + ), + ).as_workunit() + + yield from add_dataset_to_container( + container_key=self._generate_keyspace_container_key(keyspace_name), + dataset_urn=dataset_urn, + ) + + if self.config.platform_instance: + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=DataPlatformInstanceClass( + platform=make_data_platform_urn(self.platform), + instance=make_dataplatform_instance_urn( + self.platform, self.config.platform_instance + ), + ), + ).as_workunit() + + def get_upstream_fields_of_field_in_datasource( + self, table_name: str, dataset_urn: str, upstream_urn: str + ) -> List[FineGrainedLineageClass]: + column_infos = self.cassandra_data.columns.get(table_name, []) + # Collect column-level lineage + fine_grained_lineages = [] + for column_info in column_infos: + source_column = column_info.column_name + if source_column: + fine_grained_lineages.append( + FineGrainedLineageClass( + upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET, + downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD, + downstreams=[make_schema_field_urn(dataset_urn, source_column)], + upstreams=[make_schema_field_urn(upstream_urn, source_column)], + ) + ) + return fine_grained_lineages + + def get_report(self): + return self.report + + def close(self): + self.cassandra_api.close() + super().close() diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_api.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_api.py new file mode 100644 index 00000000000000..4cf0613762aab8 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_api.py @@ -0,0 +1,325 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from cassandra import DriverException, OperationTimedOut +from cassandra.auth import PlainTextAuthProvider +from cassandra.cluster import ( + EXEC_PROFILE_DEFAULT, + Cluster, + ExecutionProfile, + ProtocolVersion, + Session, +) + +from datahub.ingestion.api.source import SourceReport +from datahub.ingestion.source.cassandra.cassandra_config import CassandraSourceConfig + + +@dataclass +class CassandraKeyspace: + keyspace_name: str + durable_writes: bool + replication: Dict + + +@dataclass +class CassandraTable: + keyspace_name: str + table_name: str + bloom_filter_fp_chance: Optional[float] + caching: Optional[Dict[str, str]] + comment: Optional[str] + compaction: Optional[Dict[str, Any]] + compression: Optional[Dict[str, Any]] + crc_check_chance: Optional[float] + dclocal_read_repair_chance: Optional[float] + default_time_to_live: Optional[int] + extensions: Optional[Dict[str, Any]] + gc_grace_seconds: Optional[int] + max_index_interval: Optional[int] + memtable_flush_period_in_ms: Optional[int] + min_index_interval: Optional[int] + read_repair_chance: Optional[float] + speculative_retry: Optional[str] + + +@dataclass +class CassandraColumn: + keyspace_name: str + table_name: str + column_name: str + type: str + clustering_order: Optional[str] + kind: Optional[str] + position: Optional[int] + + +@dataclass +class CassandraView(CassandraTable): + view_name: str + include_all_columns: Optional[bool] + where_clause: str = "" + + +@dataclass +class CassandraEntities: + keyspaces: List[str] = field(default_factory=list) + tables: Dict[str, List[str]] = field( + default_factory=dict + ) # Maps keyspace -> tables + columns: Dict[str, List[CassandraColumn]] = field( + default_factory=dict + ) # Maps tables -> columns + + +# - Referencing system_schema: https://docs.datastax.com/en/cql-oss/3.x/cql/cql_using/useQuerySystem.html#Table3.ColumnsinSystem_SchemaTables-Cassandra3.0 - # +# this keyspace contains details about the cassandra cluster's keyspaces, tables, and columns + + +class CassandraQueries: + # get all keyspaces + GET_KEYSPACES_QUERY = "SELECT * FROM system_schema.keyspaces" + # get all tables for a keyspace + GET_TABLES_QUERY = "SELECT * FROM system_schema.tables WHERE keyspace_name = %s" + # get all columns for a table + GET_COLUMNS_QUERY = "SELECT * FROM system_schema.columns WHERE keyspace_name = %s AND table_name = %s" + # get all views for a keyspace + GET_VIEWS_QUERY = "SELECT * FROM system_schema.views WHERE keyspace_name = %s" + # Row Count + ROW_COUNT = 'SELECT COUNT(*) AS row_count FROM {}."{}"' + # Column Count + COLUMN_COUNT = "SELECT COUNT(*) AS column_count FROM system_schema.columns WHERE keyspace_name = '{}' AND table_name = '{}'" + + +class CassandraAPI: + def __init__(self, config: CassandraSourceConfig, report: SourceReport): + self.config = config + self.report = report + self._cassandra_session: Optional[Session] = None + + def authenticate(self) -> bool: + """Establish a connection to Cassandra and return the session.""" + try: + if self.config.cloud_config: + cloud_config = self.config.cloud_config + cluster_cloud_config = { + "connect_timeout": cloud_config.connect_timeout, + "use_default_tempdir": True, + "secure_connect_bundle": cloud_config.secure_connect_bundle, + } + profile = ExecutionProfile(request_timeout=cloud_config.request_timeout) + auth_provider = PlainTextAuthProvider( + "token", + cloud_config.token, + ) + cluster = Cluster( + cloud=cluster_cloud_config, + auth_provider=auth_provider, + execution_profiles={EXEC_PROFILE_DEFAULT: profile}, + protocol_version=ProtocolVersion.V4, + ) + + self._cassandra_session = cluster.connect() + return True + if self.config.username and self.config.password: + auth_provider = PlainTextAuthProvider( + username=self.config.username, password=self.config.password + ) + cluster = Cluster( + [self.config.contact_point], + port=self.config.port, + auth_provider=auth_provider, + load_balancing_policy=None, + ) + else: + cluster = Cluster( + [self.config.contact_point], + port=self.config.port, + load_balancing_policy=None, + ) + + self._cassandra_session = cluster.connect() + return True + except OperationTimedOut as e: + self.report.failure( + message="Failed to Authenticate", context=f"{str(e.errors)}", exc=e + ) + return False + except DriverException as e: + self.report.failure(message="Failed to Authenticate", exc=e) + return False + except Exception as e: + self.report.failure(message="Failed to authenticate to Cassandra", exc=e) + return False + + def get(self, query: str, parameters: Optional[List] = []) -> List: + if not self._cassandra_session: + return [] + + resp = self._cassandra_session.execute(query, parameters) + return resp + + def get_keyspaces(self) -> List[CassandraKeyspace]: + """Fetch all keyspaces.""" + try: + keyspaces = self.get(CassandraQueries.GET_KEYSPACES_QUERY) + keyspace_list = [ + CassandraKeyspace( + keyspace_name=row.keyspace_name, + durable_writes=row.durable_writes, + replication=dict(row.replication), + ) + for row in keyspaces + ] + return keyspace_list + except DriverException as e: + self.report.warning( + message="Failed to fetch keyspaces", context=f"{str(e)}", exc=e + ) + return [] + except Exception as e: + self.report.warning(message="Failed to fetch keyspaces", exc=e) + return [] + + def get_tables(self, keyspace_name: str) -> List[CassandraTable]: + """Fetch all tables for a given keyspace.""" + try: + tables = self.get(CassandraQueries.GET_TABLES_QUERY, [keyspace_name]) + table_list = [ + CassandraTable( + keyspace_name=row.keyspace_name, + table_name=row.table_name, + bloom_filter_fp_chance=row.bloom_filter_fp_chance, + caching=dict(row.caching), + comment=row.comment, + compaction=dict(row.compaction), + compression=dict(row.compression), + crc_check_chance=row.crc_check_chance, + dclocal_read_repair_chance=row.dclocal_read_repair_chance, + default_time_to_live=row.default_time_to_live, + extensions=dict(row.extensions), + gc_grace_seconds=row.gc_grace_seconds, + max_index_interval=row.max_index_interval, + memtable_flush_period_in_ms=row.memtable_flush_period_in_ms, + min_index_interval=row.min_index_interval, + read_repair_chance=row.read_repair_chance, + speculative_retry=row.speculative_retry, + ) + for row in tables + ] + return table_list + except DriverException as e: + self.report.warning( + message="Failed to fetch tables for keyspace", + context=f"{str(e)}", + exc=e, + ) + return [] + except Exception as e: + self.report.warning( + message="Failed to fetch tables for keyspace", + context=f"{keyspace_name}", + exc=e, + ) + return [] + + def get_columns(self, keyspace_name: str, table_name: str) -> List[CassandraColumn]: + """Fetch all columns for a given table.""" + try: + column_infos = self.get( + CassandraQueries.GET_COLUMNS_QUERY, [keyspace_name, table_name] + ) + column_list = [ + CassandraColumn( + keyspace_name=row.keyspace_name, + table_name=row.table_name, + column_name=row.column_name, + clustering_order=row.clustering_order, + kind=row.kind, + position=row.position, + type=row.type, + ) + for row in column_infos + ] + return column_list + except DriverException as e: + self.report.warning( + message="Failed to fetch columns for table", context=f"{str(e)}", exc=e + ) + return [] + except Exception as e: + self.report.warning( + message="Failed to fetch columns for table", + context=f"{keyspace_name}.{table_name}", + exc=e, + ) + return [] + + def get_views(self, keyspace_name: str) -> List[CassandraView]: + """Fetch all views for a given keyspace.""" + try: + views = self.get(CassandraQueries.GET_VIEWS_QUERY, [keyspace_name]) + view_list = [ + CassandraView( + table_name=row.base_table_name, + keyspace_name=row.keyspace_name, + view_name=row.view_name, + bloom_filter_fp_chance=row.bloom_filter_fp_chance, + caching=dict(row.caching), + comment=row.comment, + compaction=dict(row.compaction), + compression=dict(row.compression), + crc_check_chance=row.crc_check_chance, + dclocal_read_repair_chance=row.dclocal_read_repair_chance, + default_time_to_live=row.default_time_to_live, + extensions=dict(row.extensions), + gc_grace_seconds=row.gc_grace_seconds, + include_all_columns=row.include_all_columns, + max_index_interval=row.max_index_interval, + memtable_flush_period_in_ms=row.memtable_flush_period_in_ms, + min_index_interval=row.min_index_interval, + read_repair_chance=row.read_repair_chance, + speculative_retry=row.speculative_retry, + where_clause=row.where_clause, + ) + for row in views + ] + return view_list + except DriverException as e: + self.report.warning( + message="Failed to fetch views for keyspace", context=f"{str(e)}", exc=e + ) + return [] + except Exception as e: + self.report.warning( + message="Failed to fetch views for keyspace", + context=f"{keyspace_name}", + exc=e, + ) + return [] + + def execute(self, query: str, limit: Optional[int] = None) -> List: + """Fetch stats for cassandra""" + try: + if not self._cassandra_session: + return [] + if limit: + query = query + f" LIMIT {limit}" + result_set = self._cassandra_session.execute(query).all() + return result_set + except DriverException as e: + self.report.warning( + message="Failed to fetch stats for keyspace", context=str(e), exc=e + ) + return [] + except Exception: + self.report.warning( + message="Failed to fetch stats for keyspace", + context=f"{query}", + ) + return [] + + def close(self): + """Close the Cassandra session.""" + if self._cassandra_session: + self._cassandra_session.shutdown() diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_config.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_config.py new file mode 100644 index 00000000000000..340bdb68aa4585 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_config.py @@ -0,0 +1,111 @@ +from typing import Optional + +from pydantic import Field + +from datahub.configuration.common import AllowDenyPattern, ConfigModel +from datahub.configuration.source_common import ( + EnvConfigMixin, + PlatformInstanceConfigMixin, +) +from datahub.ingestion.source.ge_profiling_config import GEProfilingBaseConfig +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StatefulStaleMetadataRemovalConfig, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) +from datahub.ingestion.source_config.operation_config import is_profiling_enabled + +# - Referencing https://docs.datastax.com/en/cql-oss/3.x/cql/cql_using/useQuerySystem.html#Table3.ColumnsinSystem_SchemaTables-Cassandra3.0 - # +# this keyspace contains details about the cassandra cluster's keyspaces, tables, and columns +SYSTEM_SCHEMA_KEYSPACE_NAME = "system_schema" + +# Reference: +# https://docs.datastax.com/en/astra-db-serverless/databases/python-driver.html +# https://docs.datastax.com/en/astra-db-serverless/databases/python-driver.html#production-configuration + + +class CassandraCloudConfig(ConfigModel): + """ + Configuration for connecting to DataStax Astra DB in the cloud. + """ + + token: str = Field( + description="The Astra DB application token used for authentication.", + ) + + secure_connect_bundle: str = Field( + description="File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax Astra DB.", + ) + + connect_timeout: int = Field( + default=600, + description="Timeout in seconds for establishing new connections to Cassandra.", + ) + + request_timeout: int = Field( + default=600, description="Timeout in seconds for individual Cassandra requests." + ) + + +class CassandraSourceConfig( + PlatformInstanceConfigMixin, StatefulIngestionConfigBase, EnvConfigMixin +): + """ + Configuration for connecting to a Cassandra or DataStax Astra DB source. + """ + + contact_point: str = Field( + default="localhost", + description="Domain or IP address of the Cassandra instance (excluding port).", + ) + + port: int = Field( + default=9042, description="Port number to connect to the Cassandra instance." + ) + + username: Optional[str] = Field( + default=None, + description=f"Username credential with read access to the {SYSTEM_SCHEMA_KEYSPACE_NAME} keyspace.", + ) + + password: Optional[str] = Field( + default=None, + description="Password credential associated with the specified username.", + ) + + cloud_config: Optional[CassandraCloudConfig] = Field( + default=None, + description="Configuration for cloud-based Cassandra, such as DataStax Astra DB.", + ) + + keyspace_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter keyspaces for ingestion.", + ) + + table_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns to filter keyspaces.tables for ingestion.", + ) + + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field( + default=None, + description="Configuration for stateful ingestion and stale metadata removal.", + ) + + # Profiling + profile_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for tables to profile", + ) + + profiling: GEProfilingBaseConfig = Field( + default=GEProfilingBaseConfig(), + description="Configuration for profiling", + ) + + def is_profiling_enabled(self) -> bool: + return self.profiling.enabled and is_profiling_enabled( + self.profiling.operation_config + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_profiling.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_profiling.py new file mode 100644 index 00000000000000..d8ab62f1d6d91f --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_profiling.py @@ -0,0 +1,296 @@ +import logging +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional + +import numpy as np +from cassandra.util import OrderedMapSerializedKey, SortedSet + +from datahub.emitter.mce_builder import make_dataset_urn_with_platform_instance +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.cassandra.cassandra_api import ( + CassandraAPI, + CassandraColumn, + CassandraEntities, + CassandraQueries, +) +from datahub.ingestion.source.cassandra.cassandra_config import CassandraSourceConfig +from datahub.ingestion.source.cassandra.cassandra_utils import CassandraSourceReport +from datahub.ingestion.source_report.ingestion_stage import PROFILING +from datahub.metadata.schema_classes import ( + DatasetFieldProfileClass, + DatasetProfileClass, + QuantileClass, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class ColumnMetric: + col_type: str = "" + values: List[Any] = field(default_factory=list) + null_count: int = 0 + total_count: int = 0 + distinct_count: Optional[int] = None + min: Optional[Any] = None + max: Optional[Any] = None + mean: Optional[float] = None + stdev: Optional[float] = None + median: Optional[float] = None + quantiles: Optional[List[float]] = None + sample_values: Optional[Any] = None + + +@dataclass +class ProfileData: + row_count: Optional[int] = None + column_count: Optional[int] = None + column_metrics: Dict[str, ColumnMetric] = field(default_factory=dict) + + +class CassandraProfiler: + config: CassandraSourceConfig + report: CassandraSourceReport + + def __init__( + self, + config: CassandraSourceConfig, + report: CassandraSourceReport, + api: CassandraAPI, + ) -> None: + self.api = api + self.config = config + self.report = report + + def get_workunits( + self, cassandra_data: CassandraEntities + ) -> Iterable[MetadataWorkUnit]: + for keyspace_name in cassandra_data.keyspaces: + tables = cassandra_data.tables.get(keyspace_name, []) + self.report.set_ingestion_stage(keyspace_name, PROFILING) + with ThreadPoolExecutor( + max_workers=self.config.profiling.max_workers + ) as executor: + future_to_dataset = { + executor.submit( + self.generate_profile, + keyspace_name, + table_name, + cassandra_data.columns.get(table_name, []), + ): table_name + for table_name in tables + } + for future in as_completed(future_to_dataset): + table_name = future_to_dataset[future] + try: + yield from future.result() + except Exception as exc: + self.report.profiling_skipped_other[table_name] += 1 + self.report.failure( + message="Failed to profile for table", + context=f"{keyspace_name}.{table_name}", + exc=exc, + ) + + def generate_profile( + self, + keyspace_name: str, + table_name: str, + columns: List[CassandraColumn], + ) -> Iterable[MetadataWorkUnit]: + dataset_name: str = f"{keyspace_name}.{table_name}" + dataset_urn = make_dataset_urn_with_platform_instance( + platform="cassandra", + name=dataset_name, + env=self.config.env, + platform_instance=self.config.platform_instance, + ) + + if not columns: + self.report.warning( + message="Skipping profiling as no columns found for table", + context=f"{keyspace_name}.{table_name}", + ) + self.report.profiling_skipped_other[table_name] += 1 + return + + if not self.config.profile_pattern.allowed(f"{keyspace_name}.{table_name}"): + self.report.profiling_skipped_table_profile_pattern[keyspace_name] += 1 + logger.info( + f"Table {table_name} in {keyspace_name}, not allowed for profiling" + ) + return + + try: + profile_data = self.profile_table(keyspace_name, table_name, columns) + except Exception as e: + self.report.warning( + message="Profiling Failed", + context=f"{keyspace_name}.{table_name}", + exc=e, + ) + return + + profile_aspect = self.populate_profile_aspect(profile_data) + + if profile_aspect: + self.report.report_entity_profiled(table_name) + mcp = MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=profile_aspect + ) + yield mcp.as_workunit() + + def populate_profile_aspect(self, profile_data: ProfileData) -> DatasetProfileClass: + field_profiles = [ + self._create_field_profile(column_name, column_metrics) + for column_name, column_metrics in profile_data.column_metrics.items() + ] + return DatasetProfileClass( + timestampMillis=round(time.time() * 1000), + rowCount=profile_data.row_count, + columnCount=profile_data.column_count, + fieldProfiles=field_profiles, + ) + + def _create_field_profile( + self, field_name: str, field_stats: ColumnMetric + ) -> DatasetFieldProfileClass: + quantiles = field_stats.quantiles + return DatasetFieldProfileClass( + fieldPath=field_name, + uniqueCount=field_stats.distinct_count, + nullCount=field_stats.null_count, + min=str(field_stats.min) if field_stats.min else None, + max=str(field_stats.max) if field_stats.max else None, + mean=str(field_stats.mean) if field_stats.mean else None, + median=str(field_stats.median) if field_stats.median else None, + stdev=str(field_stats.stdev) if field_stats.stdev else None, + quantiles=[ + QuantileClass(quantile=str(0.25), value=str(quantiles[0])), + QuantileClass(quantile=str(0.75), value=str(quantiles[1])), + ] + if quantiles + else None, + sampleValues=field_stats.sample_values + if field_stats.sample_values + else None, + ) + + def profile_table( + self, keyspace_name: str, table_name: str, columns: List[CassandraColumn] + ) -> ProfileData: + profile_data = ProfileData() + + resp = self.api.execute( + CassandraQueries.ROW_COUNT.format(keyspace_name, table_name) + ) + if resp: + profile_data.row_count = resp[0].row_count + + profile_data.column_count = len(columns) + + if not self.config.profiling.profile_table_level_only: + resp = self.api.execute( + f'SELECT {", ".join([col.column_name for col in columns])} FROM {keyspace_name}."{table_name}"' + ) + profile_data.column_metrics = self._collect_column_data(resp, columns) + + return self._parse_profile_results(profile_data) + + def _parse_profile_results(self, profile_data: ProfileData) -> ProfileData: + for cl_name, column_metrics in profile_data.column_metrics.items(): + if column_metrics.values: + try: + self._compute_field_statistics(column_metrics) + except Exception as e: + self.report.warning( + message="Profiling Failed For Column Stats", + context=cl_name, + exc=e, + ) + raise e + return profile_data + + def _collect_column_data( + self, rows: List[Any], columns: List[CassandraColumn] + ) -> Dict[str, ColumnMetric]: + metrics = {column.column_name: ColumnMetric() for column in columns} + + for row in rows: + for column in columns: + if self._is_skippable_type(column.type): + continue + + value: Any = getattr(row, column.column_name, None) + metric = metrics[column.column_name] + metric.col_type = column.type + + metric.total_count += 1 + if value is None: + metric.null_count += 1 + else: + metric.values.extend(self._parse_value(value)) + + return metrics + + def _is_skippable_type(self, data_type: str) -> bool: + return data_type.lower() in ["timeuuid", "blob", "frozen>"] + + def _parse_value(self, value: Any) -> List[Any]: + if isinstance(value, SortedSet): + return list(value) + elif isinstance(value, OrderedMapSerializedKey): + return list(dict(value).values()) + elif isinstance(value, list): + return value + return [value] + + def _compute_field_statistics(self, column_metrics: ColumnMetric) -> None: + values = column_metrics.values + if not values: + return + + # ByDefault Null count is added + if not self.config.profiling.include_field_null_count: + column_metrics.null_count = 0 + + if self.config.profiling.include_field_distinct_count: + column_metrics.distinct_count = len(set(values)) + + if self.config.profiling.include_field_min_value: + column_metrics.min = min(values) + + if self.config.profiling.include_field_max_value: + column_metrics.max = max(values) + + if values and self._is_numeric_type(column_metrics.col_type): + if self.config.profiling.include_field_mean_value: + column_metrics.mean = round(float(np.mean(values)), 2) + if self.config.profiling.include_field_stddev_value: + column_metrics.stdev = round(float(np.std(values)), 2) + if self.config.profiling.include_field_median_value: + column_metrics.median = round(float(np.median(values)), 2) + if self.config.profiling.include_field_quantiles: + column_metrics.quantiles = [ + float(np.percentile(values, 25)), + float(np.percentile(values, 75)), + ] + + if values and self.config.profiling.include_field_sample_values: + column_metrics.sample_values = [str(v) for v in values[:5]] + + def _is_numeric_type(self, data_type: str) -> bool: + return data_type.lower() in [ + "int", + "counter", + "bigint", + "float", + "double", + "decimal", + "smallint", + "tinyint", + "varint", + ] diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_utils.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_utils.py new file mode 100644 index 00000000000000..41d4ac7ced6035 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra_utils.py @@ -0,0 +1,152 @@ +import logging +from dataclasses import dataclass, field +from typing import Dict, Generator, List, Optional, Type + +from datahub.ingestion.source.cassandra.cassandra_api import CassandraColumn +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, +) +from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + SchemaField, + SchemaFieldDataType, +) +from datahub.metadata.schema_classes import ( + ArrayTypeClass, + BooleanTypeClass, + BytesTypeClass, + DateTypeClass, + NullTypeClass, + NumberTypeClass, + RecordTypeClass, + StringTypeClass, + TimeTypeClass, +) +from datahub.utilities.lossy_collections import LossyList +from datahub.utilities.stats_collections import TopKDict, int_top_k_dict + +logger = logging.getLogger(__name__) + + +# we always skip over ingesting metadata about these keyspaces +SYSTEM_KEYSPACE_LIST = set( + ["system", "system_auth", "system_schema", "system_distributed", "system_traces"] +) + + +@dataclass +class CassandraSourceReport(StaleEntityRemovalSourceReport, IngestionStageReport): + num_tables_failed: int = 0 + num_views_failed: int = 0 + tables_scanned: int = 0 + views_scanned: int = 0 + entities_profiled: int = 0 + filtered: LossyList[str] = field(default_factory=LossyList) + + def report_entity_scanned(self, name: str, ent_type: str = "View") -> None: + """ + Entity could be a view or a table + """ + if ent_type == "Table": + self.tables_scanned += 1 + elif ent_type == "View": + self.views_scanned += 1 + else: + raise KeyError(f"Unknown entity {ent_type}.") + + def set_ingestion_stage(self, keyspace: str, stage: str) -> None: + self.report_ingestion_stage_start(f"{keyspace}: {stage}") + + # TODO Need to create seperate common config for profiling report + profiling_skipped_other: TopKDict[str, int] = field(default_factory=int_top_k_dict) + profiling_skipped_table_profile_pattern: TopKDict[str, int] = field( + default_factory=int_top_k_dict + ) + + def report_entity_profiled(self, name: str) -> None: + self.entities_profiled += 1 + + def report_dropped(self, ent_name: str) -> None: + self.filtered.append(ent_name) + + +# This class helps convert cassandra column types to SchemaFieldDataType for use by the datahaub metadata schema +class CassandraToSchemaFieldConverter: + # Mapping from cassandra field types to SchemaFieldDataType. + # https://cassandra.apache.org/doc/stable/cassandra/cql/types.html (version 4.1) + _field_type_to_schema_field_type: Dict[str, Type] = { + # Bool + "boolean": BooleanTypeClass, + # Binary + "blob": BytesTypeClass, + # Numbers + "bigint": NumberTypeClass, + "counter": NumberTypeClass, + "decimal": NumberTypeClass, + "double": NumberTypeClass, + "float": NumberTypeClass, + "int": NumberTypeClass, + "smallint": NumberTypeClass, + "tinyint": NumberTypeClass, + "varint": NumberTypeClass, + # Dates + "date": DateTypeClass, + # Times + "duration": TimeTypeClass, + "time": TimeTypeClass, + "timestamp": TimeTypeClass, + # Strings + "text": StringTypeClass, + "ascii": StringTypeClass, + "inet": StringTypeClass, + "timeuuid": StringTypeClass, + "uuid": StringTypeClass, + "varchar": StringTypeClass, + # Records + "geo_point": RecordTypeClass, + # Arrays + "histogram": ArrayTypeClass, + } + + @staticmethod + def get_column_type(cassandra_column_type: str) -> SchemaFieldDataType: + type_class: Optional[ + Type + ] = CassandraToSchemaFieldConverter._field_type_to_schema_field_type.get( + cassandra_column_type + ) + if type_class is None: + logger.warning( + f"Cannot map {cassandra_column_type!r} to SchemaFieldDataType, using NullTypeClass." + ) + type_class = NullTypeClass + + return SchemaFieldDataType(type=type_class()) + + def _get_schema_fields( + self, cassandra_column_infos: List[CassandraColumn] + ) -> Generator[SchemaField, None, None]: + # append each schema field (sort so output is consistent) + for column_info in cassandra_column_infos: + column_name: str = column_info.column_name + cassandra_type: str = column_info.type + + schema_field_data_type: SchemaFieldDataType = self.get_column_type( + cassandra_type + ) + schema_field: SchemaField = SchemaField( + fieldPath=column_name, + nativeDataType=cassandra_type, + type=schema_field_data_type, + description=None, + nullable=True, + recursive=False, + ) + yield schema_field + + @classmethod + def get_schema_fields( + cls, cassandra_column_infos: List[CassandraColumn] + ) -> Generator[SchemaField, None, None]: + converter = cls() + yield from converter._get_schema_fields(cassandra_column_infos) diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 7271bf6102639f..9fbb15500a863c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -40,6 +40,7 @@ class DatasetContainerSubTypes(StrEnum): S3_BUCKET = "S3 bucket" GCS_BUCKET = "GCS bucket" ABS_CONTAINER = "ABS container" + KEYSPACE = "Keyspace" # Cassandra class BIContainerSubTypes(StrEnum): diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py index 192ae7cacb8e69..9d6f65b95554e7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py @@ -1,9 +1,7 @@ -import datetime import os from typing import List, Literal, Optional import certifi -import pydantic from pydantic import Field, validator from datahub.configuration.common import AllowDenyPattern, ConfigModel @@ -11,7 +9,7 @@ EnvConfigMixin, PlatformInstanceConfigMixin, ) -from datahub.ingestion.source.ge_profiling_config import GEProfilingConfig +from datahub.ingestion.source.ge_profiling_config import GEProfilingBaseConfig from datahub.ingestion.source.state.stale_entity_removal_handler import ( StatefulStaleMetadataRemovalConfig, ) @@ -97,79 +95,15 @@ def validate_password(cls, value, values): return value -class ProfileConfig(GEProfilingConfig): - +class ProfileConfig(GEProfilingBaseConfig): query_timeout: int = Field( default=300, description="Time before cancelling Dremio profiling query" ) - - row_count: bool = True - column_count: bool = True - sample_values: bool = True - - # Below Configs inherited from GEProfilingConfig - # but not used in Dremio so we hide them from docs. include_field_median_value: bool = Field( default=False, hidden_from_docs=True, description="Median causes a number of issues in Dremio.", ) - partition_profiling_enabled: bool = Field(default=True, hidden_from_docs=True) - profile_table_row_count_estimate_only: bool = Field( - default=False, hidden_from_docs=True - ) - query_combiner_enabled: bool = Field(default=True, hidden_from_docs=True) - max_number_of_fields_to_profile: Optional[pydantic.PositiveInt] = Field( - default=None, hidden_from_docs=True - ) - profile_if_updated_since_days: Optional[pydantic.PositiveFloat] = Field( - default=None, hidden_from_docs=True - ) - profile_table_size_limit: Optional[int] = Field( - default=5, - description="Profile tables only if their size is less then specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `snowflake` and `BigQuery`", - hidden_from_docs=True, - ) - - profile_table_row_limit: Optional[int] = Field( - default=5000000, - hidden_from_docs=True, - description="Profile tables only if their row count is less then specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `snowflake` and `BigQuery`", - ) - - partition_datetime: Optional[datetime.datetime] = Field( - default=None, - hidden_from_docs=True, - description="If specified, profile only the partition which matches this datetime. " - "If not specified, profile the latest partition. Only Bigquery supports this.", - ) - use_sampling: bool = Field( - default=True, - hidden_from_docs=True, - description="Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. " - "If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", - ) - - sample_size: int = Field( - default=10000, - hidden_from_docs=True, - description="Number of rows to be sampled from table for column level profiling." - "Applicable only if `use_sampling` is set to True.", - ) - profile_external_tables: bool = Field( - default=False, - hidden_from_docs=True, - description="Whether to profile external tables. Only Snowflake and Redshift supports this.", - ) - - tags_to_ignore_sampling: Optional[List[str]] = pydantic.Field( - default=None, - hidden_from_docs=True, - description=( - "Fixed list of tags to ignore sampling." - " If not specified, tables will be sampled based on `use_sampling`." - ), - ) class DremioSourceMapping(EnvConfigMixin, PlatformInstanceConfigMixin, ConfigModel): diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_profiling.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_profiling.py index 3f25741beec670..5332597ffce9e2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_profiling.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_profiling.py @@ -149,11 +149,8 @@ def _build_profile_sql( ) -> str: metrics = [] - if self.config.profiling.row_count: - metrics.append("COUNT(*) AS row_count") - - if self.config.profiling.column_count: - metrics.append(f"{len(columns)} AS column_count") + metrics.append("COUNT(*) AS row_count") + metrics.append(f"{len(columns)} AS column_count") if not self.config.profiling.profile_table_level_only: for column_name, data_type in columns: @@ -239,11 +236,9 @@ def _parse_profile_results( profile: Dict[str, Any] = {"column_stats": {}} result = results[0] if results else {} # We expect only one row of results - if self.config.profiling.row_count: - profile["row_count"] = int(result.get("row_count", 0)) + profile["row_count"] = int(result.get("row_count", 0)) - if self.config.profiling.column_count: - profile["column_count"] = int(result.get("column_count", 0)) + profile["column_count"] = int(result.get("column_count", 0)) for column_name, data_type in columns: safe_column_name = re.sub(r"\W|^(?=\d)", "_", column_name) diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index 2a9068d3d49d8b..8b2443a589b8dc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) -class GEProfilingConfig(ConfigModel): +class GEProfilingBaseConfig(ConfigModel): enabled: bool = Field( default=False, description="Whether profiling should be done." ) @@ -35,15 +35,6 @@ class GEProfilingConfig(ConfigModel): default=None, description="Offset in documents to profile. By default, uses no offset.", ) - report_dropped_profiles: bool = Field( - default=False, - description="Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", - ) - - turn_off_expensive_profiling_metrics: bool = Field( - default=False, - description="Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", - ) profile_table_level_only: bool = Field( default=False, description="Whether to perform profiling at table-level only, or include column-level profiling as well.", @@ -92,6 +83,29 @@ class GEProfilingConfig(ConfigModel): default=True, description="Whether to profile for the sample values for all columns.", ) + + # The default of (5 * cpu_count) is adopted from the default max_workers + # parameter of ThreadPoolExecutor. Given that profiling is often an I/O-bound + # task, it may make sense to increase this default value in the future. + # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor + max_workers: int = Field( + default=5 * (os.cpu_count() or 4), + description="Number of worker threads to use for profiling. Set to 1 to disable.", + ) + + +class GEProfilingConfig(GEProfilingBaseConfig): + + report_dropped_profiles: bool = Field( + default=False, + description="Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + ) + + turn_off_expensive_profiling_metrics: bool = Field( + default=False, + description="Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + ) + field_sample_values_limit: int = Field( default=20, description="Upper limit for number of sample values to collect for all columns.", @@ -126,15 +140,6 @@ class GEProfilingConfig(ConfigModel): "less accurate. Only supported for Postgres and MySQL. ", ) - # The default of (5 * cpu_count) is adopted from the default max_workers - # parameter of ThreadPoolExecutor. Given that profiling is often an I/O-bound - # task, it may make sense to increase this default value in the future. - # https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor - max_workers: int = Field( - default=5 * (os.cpu_count() or 4), - description="Number of worker threads to use for profiling. Set to 1 to disable.", - ) - # The query combiner enables us to combine multiple queries into a single query, # reducing the number of round-trips to the database and speeding up profiling. query_combiner_enabled: bool = Field( diff --git a/metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json b/metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json new file mode 100644 index 00000000000000..1823a218ada2e0 --- /dev/null +++ b/metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json @@ -0,0 +1,2706 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "cassandra", + "env": "PROD", + "keyspace": "cass_test_1", + "durable_writes": "True", + "replication": "{\"class\": \"org.apache.cassandra.locator.SimpleStrategy\", \"replication_factor\": \"1\"}" + }, + "name": "cass_test_1", + "qualifiedName": "cass_test_1", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1731579516869, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:cassandra" + } + }, + "systemMetadata": { + "lastObserved": 1731309924399, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731309924399, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Keyspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924400, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1731309924400, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "information", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"details\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"last_updated\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"information\", \"column_name\": \"person_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "details", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_updated", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "person_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019538, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731309924405, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924405, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "information", + "qualifiedName": "cass_test_1.information", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019540, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "all_data_types", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"ascii_column\", \"type\": \"ascii\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"bigint_column\", \"type\": \"bigint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"blob_column\", \"type\": \"blob\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"boolean_column\", \"type\": \"boolean\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"date_column\", \"type\": \"date\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"decimal_column\", \"type\": \"decimal\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"double_column\", \"type\": \"double\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"float_column\", \"type\": \"float\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"frozen_list_column\", \"type\": \"frozen>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"frozen_map_column\", \"type\": \"frozen>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"frozen_set_column\", \"type\": \"frozen>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"inet_column\", \"type\": \"inet\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"int_column\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"list_column\", \"type\": \"list\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"map_column\", \"type\": \"map\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"set_column\", \"type\": \"set\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"smallint_column\", \"type\": \"smallint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"text_column\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"time_column\", \"type\": \"time\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"timestamp_column\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"timeuuid_column\", \"type\": \"timeuuid\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"tinyint_column\", \"type\": \"tinyint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"tuple_column\", \"type\": \"frozen>\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"uuid_column\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"varchar_column\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"all_data_types\", \"column_name\": \"varint_column\", \"type\": \"varint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}]" + } + }, + "fields": [ + { + "fieldPath": "ascii_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "ascii", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "bigint_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "blob_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.BytesType": {} + } + }, + "nativeDataType": "blob", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "boolean_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "date_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "date", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "decimal_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "decimal", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "double_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "float_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "frozen_list_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "frozen>", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "frozen_map_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "frozen>", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "frozen_set_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "frozen>", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "uuid", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "inet_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "inet", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "int_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "map_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "map", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "set_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "set", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "smallint_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "smallint", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "text_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "time", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "timestamp_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "timeuuid_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "timeuuid", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tinyint_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "tinyint", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "tuple_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "frozen>", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "uuid_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "uuid", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "varchar_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "varint_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "varint", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019435, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731310097192, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "people", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"cass_test_1\", \"table_name\": \"people\", \"column_name\": \"email\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"people\", \"column_name\": \"name\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_1\", \"table_name\": \"people\", \"column_name\": \"person_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "email", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "person_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019563, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731309924412, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924412, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "people", + "qualifiedName": "cass_test_1.people", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019564, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05", + "urn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924406, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05" + } + }, + "systemMetadata": { + "lastObserved": 1731309924406, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "cassandra", + "env": "PROD", + "keyspace": "cass_test_2", + "durable_writes": "True", + "replication": "{\"class\": \"org.apache.cassandra.locator.SimpleStrategy\", \"replication_factor\": \"1\"}" + }, + "name": "cass_test_2", + "qualifiedName": "cass_test_2", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1731579516849, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05", + "urn": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924413, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731309924420, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:e88cdfeb0f0ec790300527f9ea34ee05" + } + }, + "systemMetadata": { + "lastObserved": 1731309924413, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1731309924421, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "tasks", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"details\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"last_updated\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"status\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"tasks\", \"column_name\": \"task_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "details", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_updated", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "status", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "task_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019516, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731309924426, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924426, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "tasks", + "qualifiedName": "cass_test_2.tasks", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019518, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Keyspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924421, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0381892d0717b54887d087eaafd95d2b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:cassandra" + } + }, + "systemMetadata": { + "lastObserved": 1731309924420, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0381892d0717b54887d087eaafd95d2b", + "urn": "urn:li:container:0381892d0717b54887d087eaafd95d2b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731309924427, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0381892d0717b54887d087eaafd95d2b" + } + }, + "systemMetadata": { + "lastObserved": 1731309924427, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:305f73c676989511c67d97ace119138c" + } + }, + "systemMetadata": { + "lastObserved": 1731310097193, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310097193, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "task_status", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"cass_test_2\", \"table_name\": \"task_status\", \"column_name\": \"status\", \"type\": \"text\", \"clustering_order\": \"asc\", \"kind\": \"clustering\", \"position\": 0}, {\"keyspace_name\": \"cass_test_2\", \"table_name\": \"task_status\", \"column_name\": \"task_id\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "status", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "task_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019525, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:305f73c676989511c67d97ace119138c", + "urn": "urn:li:container:305f73c676989511c67d97ace119138c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310097193, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "all_data_types", + "qualifiedName": "example_keyspace.all_data_types", + "description": "Table containing all supported Cassandra data types, excluding counters", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019440, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731310097158, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310097161, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "counter_table", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"counter_table\", \"column_name\": \"counter_column\", \"type\": \"counter\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"counter_table\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "counter_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "counter", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "uuid", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019446, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "include_all_columns": "False", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "task_status", + "qualifiedName": "cass_test_2.task_status", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019524, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731310097198, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": true, + "viewLogic": "status IS NOT NULL AND task_id IS NOT NULL", + "viewLanguage": "CQL" + } + }, + "systemMetadata": { + "lastObserved": 1731310097161, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0381892d0717b54887d087eaafd95d2b" + } + }, + "systemMetadata": { + "lastObserved": 1731310097163, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0381892d0717b54887d087eaafd95d2b", + "urn": "urn:li:container:0381892d0717b54887d087eaafd95d2b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310097163, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:305f73c676989511c67d97ace119138c" + } + }, + "systemMetadata": { + "lastObserved": 1731310097199, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310097198, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD),status)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD),status)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD),task_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.task_status,PROD),task_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731447296444, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:305f73c676989511c67d97ace119138c", + "urn": "urn:li:container:305f73c676989511c67d97ace119138c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310097199, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "counter_table", + "qualifiedName": "example_keyspace.counter_table", + "description": "Separate table containing only counter column", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019447, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:cassandra" + } + }, + "systemMetadata": { + "lastObserved": 1731310097186, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "cassandra", + "env": "PROD", + "keyspace": "example_keyspace", + "durable_writes": "True", + "replication": "{\"class\": \"org.apache.cassandra.locator.SimpleStrategy\", \"replication_factor\": \"1\"}" + }, + "name": "example_keyspace", + "qualifiedName": "example_keyspace", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1731579516801, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731310097185, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Keyspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310097186, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:305f73c676989511c67d97ace119138c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1731310097186, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "shopping_cart", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"item_count\", \"type\": \"int\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"last_update_timestamp\", \"type\": \"timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"userid\", \"type\": \"text\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "item_count", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update_timestamp", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "userid", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019453, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),ascii_column)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD),ascii_column)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),bigint_column)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD),bigint_column)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD),id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731447296557, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731410842611, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731410842610, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": true, + "viewLogic": "id IS NOT NULL AND ascii_column IS NOT NULL", + "viewLanguage": "CQL" + } + }, + "systemMetadata": { + "lastObserved": 1731310103458, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "shopping_cart", + "qualifiedName": "example_keyspace.shopping_cart", + "description": "", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019455, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731310103456, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:305f73c676989511c67d97ace119138c", + "urn": "urn:li:container:305f73c676989511c67d97ace119138c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731410842612, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:305f73c676989511c67d97ace119138c" + } + }, + "systemMetadata": { + "lastObserved": 1731410842611, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310103457, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "example_view_1", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_1\", \"column_name\": \"ascii_column\", \"type\": \"ascii\", \"clustering_order\": \"asc\", \"kind\": \"clustering\", \"position\": 0}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_1\", \"column_name\": \"bigint_column\", \"type\": \"bigint\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_1\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "ascii_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "ascii", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "bigint_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "uuid", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019464, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:305f73c676989511c67d97ace119138c" + } + }, + "systemMetadata": { + "lastObserved": 1731310103460, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:305f73c676989511c67d97ace119138c", + "urn": "urn:li:container:305f73c676989511c67d97ace119138c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310103461, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "include_all_columns": "False", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "example_view_1", + "qualifiedName": "example_keyspace.example_view_1", + "description": "Example view definition with id and ascii_column", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019464, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:305f73c676989511c67d97ace119138c", + "urn": "urn:li:container:305f73c676989511c67d97ace119138c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310942175, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1731310942171, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1731310942172, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": true, + "viewLogic": "id IS NOT NULL AND ascii_column IS NOT NULL", + "viewLanguage": "CQL" + } + }, + "systemMetadata": { + "lastObserved": 1731310942172, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),ascii_column)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD),ascii_column)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),float_column)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD),float_column)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD),id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731447296594, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "bloom_filter_fp_chance": "0.01", + "caching": "{\"keys\": \"ALL\", \"rows_per_partition\": \"NONE\"}", + "compaction": "{\"class\": \"org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy\", \"max_threshold\": \"32\", \"min_threshold\": \"4\"}", + "compression": "{\"chunk_length_in_kb\": \"16\", \"class\": \"org.apache.cassandra.io.compress.LZ4Compressor\"}", + "crc_check_chance": "1.0", + "include_all_columns": "False", + "dclocal_read_repair_chance": "0.0", + "default_time_to_live": "0", + "extensions": "{}", + "gc_grace_seconds": "864000", + "max_index_interval": "2048", + "min_index_interval": "128", + "memtable_flush_period_in_ms": "0", + "read_repair_chance": "0.0", + "speculative_retry": "99p" + }, + "name": "example_view_2", + "qualifiedName": "example_keyspace.example_view_2", + "description": "Example view definition with id and ascii_column", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1731591019474, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "example_view_2", + "platform": "urn:li:dataPlatform:cassandra", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.OtherSchema": { + "rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_2\", \"column_name\": \"ascii_column\", \"type\": \"ascii\", \"clustering_order\": \"asc\", \"kind\": \"clustering\", \"position\": 0}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_2\", \"column_name\": \"float_column\", \"type\": \"float\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"example_view_2\", \"column_name\": \"id\", \"type\": \"uuid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0}]" + } + }, + "fields": [ + { + "fieldPath": "ascii_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "ascii", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "float_column", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "uuid", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731591019474, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:305f73c676989511c67d97ace119138c" + } + }, + "systemMetadata": { + "lastObserved": 1731310942175, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1731579516915, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 0, + "columnCount": 27, + "fieldProfiles": [ + { + "fieldPath": "ascii_column", + "nullCount": 0 + }, + { + "fieldPath": "bigint_column", + "nullCount": 0 + }, + { + "fieldPath": "blob_column", + "nullCount": 0 + }, + { + "fieldPath": "boolean_column", + "nullCount": 0 + }, + { + "fieldPath": "date_column", + "nullCount": 0 + }, + { + "fieldPath": "decimal_column", + "nullCount": 0 + }, + { + "fieldPath": "double_column", + "nullCount": 0 + }, + { + "fieldPath": "float_column", + "nullCount": 0 + }, + { + "fieldPath": "frozen_list_column", + "nullCount": 0 + }, + { + "fieldPath": "frozen_map_column", + "nullCount": 0 + }, + { + "fieldPath": "frozen_set_column", + "nullCount": 0 + }, + { + "fieldPath": "id", + "nullCount": 0 + }, + { + "fieldPath": "inet_column", + "nullCount": 0 + }, + { + "fieldPath": "int_column", + "nullCount": 0 + }, + { + "fieldPath": "list_column", + "nullCount": 0 + }, + { + "fieldPath": "map_column", + "nullCount": 0 + }, + { + "fieldPath": "set_column", + "nullCount": 0 + }, + { + "fieldPath": "smallint_column", + "nullCount": 0 + }, + { + "fieldPath": "text_column", + "nullCount": 0 + }, + { + "fieldPath": "time_column", + "nullCount": 0 + }, + { + "fieldPath": "timestamp_column", + "nullCount": 0 + }, + { + "fieldPath": "timeuuid_column", + "nullCount": 0 + }, + { + "fieldPath": "tinyint_column", + "nullCount": 0 + }, + { + "fieldPath": "tuple_column", + "nullCount": 0 + }, + { + "fieldPath": "uuid_column", + "nullCount": 0 + }, + { + "fieldPath": "varchar_column", + "nullCount": 0 + }, + { + "fieldPath": "varint_column", + "nullCount": 0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731579516925, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1731579516959, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 0, + "columnCount": 3, + "fieldProfiles": [ + { + "fieldPath": "email", + "nullCount": 0 + }, + { + "fieldPath": "name", + "nullCount": 0 + }, + { + "fieldPath": "person_id", + "nullCount": 0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731579516960, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1731579516904, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 0, + "columnCount": 2, + "fieldProfiles": [ + { + "fieldPath": "counter_column", + "nullCount": 0 + }, + { + "fieldPath": "id", + "nullCount": 0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731579516915, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1731579516939, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 0, + "columnCount": 4, + "fieldProfiles": [ + { + "fieldPath": "details", + "nullCount": 0 + }, + { + "fieldPath": "last_updated", + "nullCount": 0 + }, + { + "fieldPath": "status", + "nullCount": 0 + }, + { + "fieldPath": "task_id", + "nullCount": 0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731579516950, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1731579516950, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 0, + "columnCount": 3, + "fieldProfiles": [ + { + "fieldPath": "details", + "nullCount": 0 + }, + { + "fieldPath": "last_updated", + "nullCount": 0 + }, + { + "fieldPath": "person_id", + "nullCount": 0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731579516959, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1731579516925, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 9, + "columnCount": 3, + "fieldProfiles": [ + { + "fieldPath": "item_count", + "uniqueCount": 5, + "nullCount": 4, + "min": "2", + "max": "100", + "mean": "46.4", + "median": "50.0", + "stdev": "38.44", + "sampleValues": [ + "5", + "100", + "75", + "2", + "50" + ] + }, + { + "fieldPath": "last_update_timestamp", + "uniqueCount": 9, + "nullCount": 0, + "min": "2024-11-01 00:00:00", + "max": "2024-11-09 00:00:00", + "sampleValues": [ + "2024-11-08 00:00:00", + "2024-11-06 00:00:00", + "2024-11-02 00:00:00", + "2024-11-03 00:00:00", + "2024-11-05 00:00:00" + ] + }, + { + "fieldPath": "userid", + "uniqueCount": 9, + "nullCount": 0, + "min": "1234", + "max": "9876", + "sampleValues": [ + "1240", + "1238", + "1234", + "1235", + "1237" + ] + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1731579516939, + "runId": "cassandra-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/cassandra/docker-compose.yml b/metadata-ingestion/tests/integration/cassandra/docker-compose.yml new file mode 100644 index 00000000000000..a1a2a3b97d134b --- /dev/null +++ b/metadata-ingestion/tests/integration/cassandra/docker-compose.yml @@ -0,0 +1,38 @@ +version: "1" +services: + test-cassandra: + image: cassandra:latest + container_name: test-cassandra + ports: + - 9042:9042 + volumes: + - ./setup/cassandra.yaml:/etc/cassandra/cassandra.yaml + - ./setup/init_keyspaces.cql:/docker-entrypoint-initdb.d/init_keyspaces.cql + networks: + - testnet + healthcheck: + test: ["CMD-SHELL", "cqlsh -e 'describe keyspaces' || exit 1"] + interval: 10s + timeout: 10s + retries: 10 + + test-cassandra-load-keyspace: + container_name: test-cassandra-load-keyspace + image: cassandra:latest + depends_on: + test-cassandra: + condition: service_healthy + volumes: + - ./setup/init_keyspaces.cql:/init_keyspaces.cql + command: /bin/bash -c "echo loading cassandra keyspace && cqlsh test-cassandra -f init_keyspaces.cql" + deploy: + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 100s + networks: + - testnet + +networks: + testnet: diff --git a/metadata-ingestion/tests/integration/cassandra/setup/cassandra.yaml b/metadata-ingestion/tests/integration/cassandra/setup/cassandra.yaml new file mode 100644 index 00000000000000..2dee71a0b5d1bb --- /dev/null +++ b/metadata-ingestion/tests/integration/cassandra/setup/cassandra.yaml @@ -0,0 +1,1827 @@ +# ---------------------------------------------------------------------------------------- # +# NOTE: +# This yaml has been copied from the official Cassandra Docker image: +# (https://github.com/datastax/docker-images/blob/master/config-templates/DSE/6.0.0/cassandra.yaml) +# +# Key interests are in setting the AllowAllNetworkAuthorizer, +# and in enabling materialized view feature. +# ---------------------------------------------------------------------------------------- # + +# Cassandra storage config YAML + +# NOTE: +# See https://cassandra.apache.org/doc/latest/configuration/ for +# full explanations of configuration directives +# /NOTE + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +cluster_name: 'Test Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +# +# If you leave this unspecified, Cassandra will use the default of 1 token for legacy compatibility, +# and will use the initial_token as described below. +# +# Specifying initial_token will override this setting on the node's initial start, +# on subsequent starts, this setting will apply even if initial token is set. +# +# See https://cassandra.apache.org/doc/latest/getting_started/production.html#tokens for +# best practice information about num_tokens. +# +num_tokens: 16 + +# Triggers automatic allocation of num_tokens tokens for this node. The allocation +# algorithm attempts to choose tokens in a way that optimizes replicated load over +# the nodes in the datacenter for the replica factor. +# +# The load assigned to each node will be close to proportional to its number of +# vnodes. +# +# Only supported with the Murmur3Partitioner. + +# Replica factor is determined via the replication strategy used by the specified +# keyspace. +# allocate_tokens_for_keyspace: KEYSPACE + +# Replica factor is explicitly set, regardless of keyspace or datacenter. +# This is the replica factor within the datacenter, like NTS. +allocate_tokens_for_local_replication_factor: 3 + +# initial_token allows you to specify tokens manually. While you can use it with +# vnodes (num_tokens > 1, above) -- in which case you should provide a +# comma-separated list -- it's primarily used when adding nodes to legacy clusters +# that do not have vnodes enabled. +# initial_token: + +# May either be "true" or "false" to enable globally +hinted_handoff_enabled: true + +# When hinted_handoff_enabled is true, a black list of data centers that will not +# perform hinted handoff +# hinted_handoff_disabled_datacenters: +# - DC1 +# - DC2 + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +# Min unit: ms +max_hint_window: 3h + +# Maximum throttle in KiBs per second, per delivery thread. This will be +# reduced proportionally to the number of nodes in the cluster. (If there +# are two nodes in the cluster, each delivery thread will use the maximum +# rate; if there are three, each will throttle to half of the maximum, +# since we expect two nodes to be delivering hints simultaneously.) +# Min unit: KiB +hinted_handoff_throttle: 1024KiB + +# Number of threads with which to deliver hints; +# Consider increasing this number when you have multi-dc deployments, since +# cross-dc handoff tends to be slower +max_hints_delivery_threads: 2 + +# Directory where Cassandra should store hints. +# If not set, the default directory is $CASSANDRA_HOME/data/hints. +# hints_directory: /var/lib/cassandra/hints + +# How often hints should be flushed from the internal buffers to disk. +# Will *not* trigger fsync. +# Min unit: ms +hints_flush_period: 10000ms + +# Maximum size for a single hints file, in mebibytes. +# Min unit: MiB +max_hints_file_size: 128MiB + +# The file size limit to store hints for an unreachable host, in mebibytes. +# Once the local hints files have reached the limit, no more new hints will be created. +# Set a non-positive value will disable the size limit. +# max_hints_size_per_host: 0MiB + +# Enable / disable automatic cleanup for the expired and orphaned hints file. +# Disable the option in order to preserve those hints on the disk. +auto_hints_cleanup_enabled: false + +# Compression to apply to the hint files. If omitted, hints files +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +#hints_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# Enable / disable persistent hint windows. +# +# If set to false, a hint will be stored only in case a respective node +# that hint is for is down less than or equal to max_hint_window. +# +# If set to true, a hint will be stored in case there is not any +# hint which was stored earlier than max_hint_window. This is for cases +# when a node keeps to restart and hints are not delivered yet, we would be saving +# hints for that node indefinitely. +# +# Defaults to true. +# +# hint_window_persistent_enabled: true + +# Maximum throttle in KiBs per second, total. This will be +# reduced proportionally to the number of nodes in the cluster. +# Min unit: KiB +batchlog_replay_throttle: 1024KiB + +# Authentication backend, implementing IAuthenticator; used to identify users +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthenticator, +# PasswordAuthenticator}. +# +# - AllowAllAuthenticator performs no checks - set it to disable authentication. +# - PasswordAuthenticator relies on username/password pairs to authenticate +# users. It keeps usernames and hashed passwords in system_auth.roles table. +# Please increase system_auth keyspace replication factor if you use this authenticator. +# If using PasswordAuthenticator, CassandraRoleManager must also be used (see below) +authenticator: AllowAllAuthenticator + +# Authorization backend, implementing IAuthorizer; used to limit access/provide permissions +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllAuthorizer, +# CassandraAuthorizer}. +# +# - AllowAllAuthorizer allows any action to any user - set it to disable authorization. +# - CassandraAuthorizer stores permissions in system_auth.role_permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +authorizer: AllowAllAuthorizer + +# Part of the Authentication & Authorization backend, implementing IRoleManager; used +# to maintain grants and memberships between roles. +# Out of the box, Cassandra provides org.apache.cassandra.auth.CassandraRoleManager, +# which stores role information in the system_auth keyspace. Most functions of the +# IRoleManager require an authenticated login, so unless the configured IAuthenticator +# actually implements authentication, most of this functionality will be unavailable. +# +# - CassandraRoleManager stores role data in the system_auth keyspace. Please +# increase system_auth keyspace replication factor if you use this role manager. +role_manager: CassandraRoleManager + +# Network authorization backend, implementing INetworkAuthorizer; used to restrict user +# access to certain DCs +# Out of the box, Cassandra provides org.apache.cassandra.auth.{AllowAllNetworkAuthorizer, +# CassandraNetworkAuthorizer}. +# +# - AllowAllNetworkAuthorizer allows access to any DC to any user - set it to disable authorization. +# - CassandraNetworkAuthorizer stores permissions in system_auth.network_permissions table. Please +# increase system_auth keyspace replication factor if you use this authorizer. +network_authorizer: AllowAllNetworkAuthorizer + +# Depending on the auth strategy of the cluster, it can be beneficial to iterate +# from root to table (root -> ks -> table) instead of table to root (table -> ks -> root). +# As the auth entries are whitelisting, once a permission is found you know it to be +# valid. We default to false as the legacy behavior is to query at the table level then +# move back up to the root. See CASSANDRA-17016 for details. +# traverse_auth_from_root: false + +# Validity period for roles cache (fetching granted roles can be an expensive +# operation depending on the role manager, CassandraRoleManager is one example) +# Granted roles are cached for authenticated sessions in AuthenticatedUser and +# after the period specified here, become eligible for (async) reload. +# Defaults to 2000, set to 0 to disable caching entirely. +# Will be disabled automatically for AllowAllAuthenticator. +# For a long-running cache using roles_cache_active_update, consider +# setting to something longer such as a daily validation: 86400000 +# Min unit: ms +roles_validity: 2000ms + +# Refresh interval for roles cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If roles_validity is non-zero, then this must be +# also. +# This setting is also used to inform the interval of auto-updating if +# using roles_cache_active_update. +# Defaults to the same value as roles_validity. +# For a long-running cache, consider setting this to 60000 (1 hour) etc. +# Min unit: ms +# roles_update_interval: 2000ms + +# If true, cache contents are actively updated by a background task at the +# interval set by roles_update_interval. If false, cache entries +# become eligible for refresh after their update interval. Upon next access, +# an async reload is scheduled and the old value returned until it completes. +# roles_cache_active_update: false + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 2000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +# For a long-running cache using permissions_cache_active_update, consider +# setting to something longer such as a daily validation: 86400000ms +# Min unit: ms +permissions_validity: 2000ms + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity is non-zero, then this must be +# also. +# This setting is also used to inform the interval of auto-updating if +# using permissions_cache_active_update. +# Defaults to the same value as permissions_validity. +# For a longer-running permissions cache, consider setting to update hourly (60000) +# Min unit: ms +# permissions_update_interval: 2000ms + +# If true, cache contents are actively updated by a background task at the +# interval set by permissions_update_interval. If false, cache entries +# become eligible for refresh after their update interval. Upon next access, +# an async reload is scheduled and the old value returned until it completes. +# permissions_cache_active_update: false + +# Validity period for credentials cache. This cache is tightly coupled to +# the provided PasswordAuthenticator implementation of IAuthenticator. If +# another IAuthenticator implementation is configured, this cache will not +# be automatically used and so the following settings will have no effect. +# Please note, credentials are cached in their encrypted form, so while +# activating this cache may reduce the number of queries made to the +# underlying table, it may not bring a significant reduction in the +# latency of individual authentication attempts. +# Defaults to 2000, set to 0 to disable credentials caching. +# For a long-running cache using credentials_cache_active_update, consider +# setting to something longer such as a daily validation: 86400000 +# Min unit: ms +credentials_validity: 2000ms + +# Refresh interval for credentials cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If credentials_validity is non-zero, then this must be +# also. +# This setting is also used to inform the interval of auto-updating if +# using credentials_cache_active_update. +# Defaults to the same value as credentials_validity. +# For a longer-running permissions cache, consider setting to update hourly (60000) +# Min unit: ms +# credentials_update_interval: 2000ms + +# If true, cache contents are actively updated by a background task at the +# interval set by credentials_update_interval. If false (default), cache entries +# become eligible for refresh after their update interval. Upon next access, +# an async reload is scheduled and the old value returned until it completes. +# credentials_cache_active_update: false + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. The partitioner can NOT be +# changed without reloading all data. If you are adding nodes or upgrading, +# you should set this to the same partitioner that you are currently using. +# +# The default partitioner is the Murmur3Partitioner. Older partitioners +# such as the RandomPartitioner, ByteOrderedPartitioner, and +# OrderPreservingPartitioner have been included for backward compatibility only. +# For new clusters, you should NOT change this value. +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Directories where Cassandra should store data on disk. If multiple +# directories are specified, Cassandra will spread data evenly across +# them by partitioning the token ranges. +# If not set, the default directory is $CASSANDRA_HOME/data/data. +# data_file_directories: +# - /var/lib/cassandra/data + +# Directory were Cassandra should store the data of the local system keyspaces. +# By default Cassandra will store the data of the local system keyspaces in the first of the data directories specified +# by data_file_directories. +# This approach ensures that if one of the other disks is lost Cassandra can continue to operate. For extra security +# this setting allows to store those data on a different directory that provides redundancy. +# local_system_data_file_directory: + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +# If not set, the default directory is $CASSANDRA_HOME/data/commitlog. +# commitlog_directory: /var/lib/cassandra/commitlog + +# Enable / disable CDC functionality on a per-node basis. This modifies the logic used +# for write path allocation rejection (standard: never reject. cdc: reject Mutation +# containing a CDC-enabled table if at space limit in cdc_raw_directory). +cdc_enabled: false + +# CommitLogSegments are moved to this directory on flush if cdc_enabled: true and the +# segment contains mutations for a CDC-enabled table. This should be placed on a +# separate spindle than the data directories. If not set, the default directory is +# $CASSANDRA_HOME/data/cdc_raw. +# cdc_raw_directory: /var/lib/cassandra/cdc_raw + +# Policy for data disk failures: +# +# die +# shut down gossip and client transports and kill the JVM for any fs errors or +# single-sstable errors, so the node can be replaced. +# +# stop_paranoid +# shut down gossip and client transports even for single-sstable errors, +# kill the JVM for errors during startup. +# +# stop +# shut down gossip and client transports, leaving the node effectively dead, but +# can still be inspected via JMX, kill the JVM for errors during startup. +# +# best_effort +# stop using the failed disk and respond to requests based on +# remaining available sstables. This means you WILL see obsolete +# data at CL.ONE! +# +# ignore +# ignore fatal errors and let requests fail, as in pre-1.2 Cassandra +disk_failure_policy: stop + +# Policy for commit disk failures: +# +# die +# shut down the node and kill the JVM, so the node can be replaced. +# +# stop +# shut down the node, leaving the node effectively dead, but +# can still be inspected via JMX. +# +# stop_commit +# shutdown the commit log, letting writes collect but +# continuing to service reads, as in pre-2.0.5 Cassandra +# +# ignore +# ignore fatal errors and let the batches fail +commit_failure_policy: stop + +# Maximum size of the native protocol prepared statement cache +# +# Valid values are either "auto" (omitting the value) or a value greater 0. +# +# Note that specifying a too large value will result in long running GCs and possbily +# out-of-memory errors. Keep the value at a small fraction of the heap. +# +# If you constantly see "prepared statements discarded in the last minute because +# cache limit reached" messages, the first step is to investigate the root cause +# of these messages and check whether prepared statements are used correctly - +# i.e. use bind markers for variable parts. +# +# Do only change the default value, if you really have more prepared statements than +# fit in the cache. In most cases it is not neccessary to change this value. +# Constantly re-preparing statements is a performance penalty. +# +# Default value ("auto") is 1/256th of the heap or 10MiB, whichever is greater +# Min unit: MiB +prepared_statements_cache_size: + +# Maximum size of the key cache in memory. +# +# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the +# minimum, sometimes more. The key cache is fairly tiny for the amount of +# time it saves, so it's worthwhile to use it at large numbers. +# The row cache saves even more time, but must contain the entire row, +# so it is extremely space-intensive. It's best to only use the +# row cache if you have hot rows or static rows. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(5% of Heap (in MiB), 100MiB)). Set to 0 to disable key cache. +# Min unit: MiB +key_cache_size: + +# Duration in seconds after which Cassandra should +# save the key cache. Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 14400 or 4 hours. +# Min unit: s +key_cache_save_period: 4h + +# Number of keys from the key cache to save +# Disabled by default, meaning all keys are going to be saved +# key_cache_keys_to_save: 100 + +# Row cache implementation class name. Available implementations: +# +# org.apache.cassandra.cache.OHCProvider +# Fully off-heap row cache implementation (default). +# +# org.apache.cassandra.cache.SerializingCacheProvider +# This is the row cache implementation availabile +# in previous releases of Cassandra. +# row_cache_class_name: org.apache.cassandra.cache.OHCProvider + +# Maximum size of the row cache in memory. +# Please note that OHC cache implementation requires some additional off-heap memory to manage +# the map structures and some in-flight memory during operations before/after cache entries can be +# accounted against the cache capacity. This overhead is usually small compared to the whole capacity. +# Do not specify more memory that the system can afford in the worst usual situation and leave some +# headroom for OS block level cache. Do never allow your system to swap. +# +# Default value is 0, to disable row caching. +# Min unit: MiB +row_cache_size: 0MiB + +# Duration in seconds after which Cassandra should save the row cache. +# Caches are saved to saved_caches_directory as specified in this configuration file. +# +# Saved caches greatly improve cold-start speeds, and is relatively cheap in +# terms of I/O for the key cache. Row cache saving is much more expensive and +# has limited use. +# +# Default is 0 to disable saving the row cache. +# Min unit: s +row_cache_save_period: 0s + +# Number of keys from the row cache to save. +# Specify 0 (which is the default), meaning all keys are going to be saved +# row_cache_keys_to_save: 100 + +# Maximum size of the counter cache in memory. +# +# Counter cache helps to reduce counter locks' contention for hot counter cells. +# In case of RF = 1 a counter cache hit will cause Cassandra to skip the read before +# write entirely. With RF > 1 a counter cache hit will still help to reduce the duration +# of the lock hold, helping with hot counter cell updates, but will not allow skipping +# the read entirely. Only the local (clock, count) tuple of a counter cell is kept +# in memory, not the whole counter, so it's relatively cheap. +# +# NOTE: if you reduce the size, you may not get you hottest keys loaded on startup. +# +# Default value is empty to make it "auto" (min(2.5% of Heap (in MiB), 50MiB)). Set to 0 to disable counter cache. +# NOTE: if you perform counter deletes and rely on low gcgs, you should disable the counter cache. +# Min unit: MiB +counter_cache_size: + +# Duration in seconds after which Cassandra should +# save the counter cache (keys only). Caches are saved to saved_caches_directory as +# specified in this configuration file. +# +# Default is 7200 or 2 hours. +# Min unit: s +counter_cache_save_period: 7200s + +# Number of keys from the counter cache to save +# Disabled by default, meaning all keys are going to be saved +# counter_cache_keys_to_save: 100 + +# saved caches +# If not set, the default directory is $CASSANDRA_HOME/data/saved_caches. +# saved_caches_directory: /var/lib/cassandra/saved_caches + +# Number of seconds the server will wait for each cache (row, key, etc ...) to load while starting +# the Cassandra process. Setting this to zero is equivalent to disabling all cache loading on startup +# while still having the cache during runtime. +# Min unit: s +# cache_load_timeout: 30s + +# commitlog_sync may be either "periodic", "group", or "batch." +# +# When in batch mode, Cassandra won't ack writes until the commit log +# has been flushed to disk. Each incoming write will trigger the flush task. +# commitlog_sync_batch_window_in_ms is a deprecated value. Previously it had +# almost no value, and is being removed. +# +# commitlog_sync_batch_window_in_ms: 2 +# +# group mode is similar to batch mode, where Cassandra will not ack writes +# until the commit log has been flushed to disk. The difference is group +# mode will wait up to commitlog_sync_group_window between flushes. +# +# Min unit: ms +# commitlog_sync_group_window: 1000ms +# +# the default option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period +# milliseconds. +commitlog_sync: periodic +# Min unit: ms +commitlog_sync_period: 10000ms + +# When in periodic commitlog mode, the number of milliseconds to block writes +# while waiting for a slow disk flush to complete. +# Min unit: ms +# periodic_commitlog_sync_lag_block: + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +# Max mutation size is also configurable via max_mutation_size setting in +# cassandra.yaml. The default is half the size commitlog_segment_size in bytes. +# This should be positive and less than 2048. +# +# NOTE: If max_mutation_size is set explicitly then commitlog_segment_size must +# be set to at least twice the size of max_mutation_size +# +# Min unit: MiB +commitlog_segment_size: 32MiB + +# Compression to apply to the commit log. If omitted, the commit log +# will be written uncompressed. LZ4, Snappy, and Deflate compressors +# are supported. +# commitlog_compression: +# - class_name: LZ4Compressor +# parameters: +# - + +# Compression to apply to SSTables as they flush for compressed tables. +# Note that tables without compression enabled do not respect this flag. +# +# As high ratio compressors like LZ4HC, Zstd, and Deflate can potentially +# block flushes for too long, the default is to flush with a known fast +# compressor in those cases. Options are: +# +# none : Flush without compressing blocks but while still doing checksums. +# fast : Flush with a fast compressor. If the table is already using a +# fast compressor that compressor is used. +# table: Always flush with the same compressor that the table uses. This +# was the pre 4.0 behavior. +# +# flush_compression: fast + +# any class that implements the SeedProvider interface and has a +# constructor that takes a Map of parameters will do. +seed_provider: + # Addresses of hosts that are deemed contact points. + # Cassandra nodes use this list of hosts to find each other and learn + # the topology of the ring. You must change this if you are running + # multiple nodes! + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # seeds is actually a comma-delimited list of addresses. + # Ex: ",," + - seeds: "172.18.0.2" + +# For workloads with more data than can fit in memory, Cassandra's +# bottleneck will be reads that need to fetch data from +# disk. "concurrent_reads" should be set to (16 * number_of_drives) in +# order to allow the operations to enqueue low enough in the stack +# that the OS and drives can reorder them. Same applies to +# "concurrent_counter_writes", since counter writes read the current +# values before incrementing and writing them back. +# +# On the other hand, since writes are almost never IO bound, the ideal +# number of "concurrent_writes" is dependent on the number of cores in +# your system; (8 * number_of_cores) is a good rule of thumb. +concurrent_reads: 32 +concurrent_writes: 32 +concurrent_counter_writes: 32 + +# For materialized view writes, as there is a read involved, so this should +# be limited by the less of concurrent reads or concurrent writes. +concurrent_materialized_view_writes: 32 + +# Maximum memory to use for inter-node and client-server networking buffers. +# +# Defaults to the smaller of 1/16 of heap or 128MB. This pool is allocated off-heap, +# so is in addition to the memory allocated for heap. The cache also has on-heap +# overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size +# if the default 64k chunk size is used). +# Memory is only allocated when needed. +# Min unit: MiB +# networking_cache_size: 128MiB + +# Enable the sstable chunk cache. The chunk cache will store recently accessed +# sections of the sstable in-memory as uncompressed buffers. +# file_cache_enabled: false + +# Maximum memory to use for sstable chunk cache and buffer pooling. +# 32MB of this are reserved for pooling buffers, the rest is used for chunk cache +# that holds uncompressed sstable chunks. +# Defaults to the smaller of 1/4 of heap or 512MB. This pool is allocated off-heap, +# so is in addition to the memory allocated for heap. The cache also has on-heap +# overhead which is roughly 128 bytes per chunk (i.e. 0.2% of the reserved size +# if the default 64k chunk size is used). +# Memory is only allocated when needed. +# Min unit: MiB +# file_cache_size: 512MiB + +# Flag indicating whether to allocate on or off heap when the sstable buffer +# pool is exhausted, that is when it has exceeded the maximum memory +# file_cache_size, beyond which it will not cache buffers but allocate on request. + +# buffer_pool_use_heap_if_exhausted: true + +# The strategy for optimizing disk read +# Possible values are: +# ssd (for solid state disks, the default) +# spinning (for spinning disks) +# disk_optimization_strategy: ssd + +# Total permitted memory to use for memtables. Cassandra will stop +# accepting writes when the limit is exceeded until a flush completes, +# and will trigger a flush based on memtable_cleanup_threshold +# If omitted, Cassandra will set both to 1/4 the size of the heap. +# Min unit: MiB +# memtable_heap_space: 2048MiB +# Min unit: MiB +# memtable_offheap_space: 2048MiB + +# memtable_cleanup_threshold is deprecated. The default calculation +# is the only reasonable choice. See the comments on memtable_flush_writers +# for more information. +# +# Ratio of occupied non-flushing memtable size to total permitted size +# that will trigger a flush of the largest memtable. Larger mct will +# mean larger flushes and hence less compaction, but also less concurrent +# flush activity which can make it difficult to keep your disks fed +# under heavy write load. +# +# memtable_cleanup_threshold defaults to 1 / (memtable_flush_writers + 1) +# memtable_cleanup_threshold: 0.11 + +# Specify the way Cassandra allocates and manages memtable memory. +# Options are: +# +# heap_buffers +# on heap nio buffers +# +# offheap_buffers +# off heap (direct) nio buffers +# +# offheap_objects +# off heap objects +memtable_allocation_type: heap_buffers + +# Limit memory usage for Merkle tree calculations during repairs. The default +# is 1/16th of the available heap. The main tradeoff is that smaller trees +# have less resolution, which can lead to over-streaming data. If you see heap +# pressure during repairs, consider lowering this, but you cannot go below +# one mebibyte. If you see lots of over-streaming, consider raising +# this or using subrange repair. +# +# For more details see https://issues.apache.org/jira/browse/CASSANDRA-14096. +# +# Min unit: MiB +# repair_session_space: + +# Total space to use for commit logs on disk. +# +# If space gets above this value, Cassandra will flush every dirty CF +# in the oldest segment and remove it. So a small total commitlog space +# will tend to cause more flush activity on less-active columnfamilies. +# +# The default value is the smaller of 8192, and 1/4 of the total space +# of the commitlog volume. +# +# commitlog_total_space: 8192MiB + +# This sets the number of memtable flush writer threads per disk +# as well as the total number of memtables that can be flushed concurrently. +# These are generally a combination of compute and IO bound. +# +# Memtable flushing is more CPU efficient than memtable ingest and a single thread +# can keep up with the ingest rate of a whole server on a single fast disk +# until it temporarily becomes IO bound under contention typically with compaction. +# At that point you need multiple flush threads. At some point in the future +# it may become CPU bound all the time. +# +# You can tell if flushing is falling behind using the MemtablePool.BlockedOnAllocation +# metric which should be 0, but will be non-zero if threads are blocked waiting on flushing +# to free memory. +# +# memtable_flush_writers defaults to two for a single data directory. +# This means that two memtables can be flushed concurrently to the single data directory. +# If you have multiple data directories the default is one memtable flushing at a time +# but the flush will use a thread per data directory so you will get two or more writers. +# +# Two is generally enough to flush on a fast disk [array] mounted as a single data directory. +# Adding more flush writers will result in smaller more frequent flushes that introduce more +# compaction overhead. +# +# There is a direct tradeoff between number of memtables that can be flushed concurrently +# and flush size and frequency. More is not better you just need enough flush writers +# to never stall waiting for flushing to free memory. +# +# memtable_flush_writers: 2 + +# Total space to use for change-data-capture logs on disk. +# +# If space gets above this value, Cassandra will throw WriteTimeoutException +# on Mutations including tables with CDC enabled. A CDCCompactor is responsible +# for parsing the raw CDC logs and deleting them when parsing is completed. +# +# The default value is the min of 4096 MiB and 1/8th of the total space +# of the drive where cdc_raw_directory resides. +# Min unit: MiB +# cdc_total_space: 4096MiB + +# When we hit our cdc_raw limit and the CDCCompactor is either running behind +# or experiencing backpressure, we check at the following interval to see if any +# new space for cdc-tracked tables has been made available. Default to 250ms +# Min unit: ms +# cdc_free_space_check_interval: 250ms + +# A fixed memory pool size in MB for for SSTable index summaries. If left +# empty, this will default to 5% of the heap size. If the memory usage of +# all index summaries exceeds this limit, SSTables with low read rates will +# shrink their index summaries in order to meet this limit. However, this +# is a best-effort process. In extreme conditions Cassandra may need to use +# more than this amount of memory. +# Min unit: KiB +index_summary_capacity: + +# How frequently index summaries should be resampled. This is done +# periodically to redistribute memory from the fixed-size pool to sstables +# proportional their recent read rates. Setting to null value will disable this +# process, leaving existing index summaries at their current sampling level. +# Min unit: m +index_summary_resize_interval: 60m + +# Whether to, when doing sequential writing, fsync() at intervals in +# order to force the operating system to flush the dirty +# buffers. Enable this to avoid sudden dirty buffer flushing from +# impacting read latencies. Almost always a good idea on SSDs; not +# necessarily on platters. +trickle_fsync: false +# Min unit: KiB +trickle_fsync_interval: 10240KiB + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +storage_port: 7000 + +# SSL port, for legacy encrypted communication. This property is unused unless enabled in +# server_encryption_options (see below). As of cassandra 4.0, this property is deprecated +# as a single port can be used for either/both secure and insecure connections. +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +ssl_storage_port: 7001 + +# Address or interface to bind to and tell other Cassandra nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# Set listen_address OR listen_interface, not both. +# +# Leaving it blank leaves it up to InetAddress.getLocalHost(). This +# will always do the Right Thing _if_ the node is properly configured +# (hostname, name resolution, etc), and the Right Thing is to use the +# address associated with the hostname (it might not be). If unresolvable +# it will fall back to InetAddress.getLoopbackAddress(), which is wrong for production systems. +# +# Setting listen_address to 0.0.0.0 is always wrong. +# +listen_address: 172.18.0.2 + +# Set listen_address OR listen_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# listen_interface: eth0 + +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using listen_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +# listen_interface_prefer_ipv6: false + +# Address to broadcast to other Cassandra nodes +# Leaving this blank will set it to the same value as listen_address +broadcast_address: 172.18.0.2 + +# When using multiple physical network interfaces, set this +# to true to listen on broadcast_address in addition to +# the listen_address, allowing nodes to communicate in both +# interfaces. +# Ignore this property if the network configuration automatically +# routes between the public and private networks such as EC2. +# listen_on_broadcast_address: false + +# Internode authentication backend, implementing IInternodeAuthenticator; +# used to allow/disallow connections from peer nodes. +# internode_authenticator: org.apache.cassandra.auth.AllowAllInternodeAuthenticator + +# Whether to start the native transport server. +# The address on which the native transport is bound is defined by rpc_address. +start_native_transport: true +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +native_transport_port: 9042 +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. +# native_transport_port_ssl: 9142 +# The maximum threads for handling requests (note that idle threads are stopped +# after 30 seconds so there is not corresponding minimum setting). +# native_transport_max_threads: 128 +# +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 16MiB. If you're changing this parameter, +# you may want to adjust max_value_size accordingly. This should be positive and less than 2048. +# Min unit: MiB +# native_transport_max_frame_size: 16MiB + +# The maximum number of concurrent client connections. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections: -1 + +# The maximum number of concurrent client connections per source ip. +# The default is -1, which means unlimited. +# native_transport_max_concurrent_connections_per_ip: -1 + +# Controls whether Cassandra honors older, yet currently supported, protocol versions. +# The default is true, which means all supported protocols will be honored. +native_transport_allow_older_protocols: true + +# Controls when idle client connections are closed. Idle connections are ones that had neither reads +# nor writes for a time period. +# +# Clients may implement heartbeats by sending OPTIONS native protocol message after a timeout, which +# will reset idle timeout timer on the server side. To close idle client connections, corresponding +# values for heartbeat intervals have to be set on the client side. +# +# Idle connection timeouts are disabled by default. +# Min unit: ms +# native_transport_idle_timeout: 60000ms + +# When enabled, limits the number of native transport requests dispatched for processing per second. +# Behavior once the limit has been breached depends on the value of THROW_ON_OVERLOAD specified in +# the STARTUP message sent by the client during connection establishment. (See section "4.1.1. STARTUP" +# in "CQL BINARY PROTOCOL v5".) With the THROW_ON_OVERLOAD flag enabled, messages that breach the limit +# are dropped, and an OverloadedException is thrown for the client to handle. When the flag is not +# enabled, the server will stop consuming messages from the channel/socket, putting backpressure on +# the client while already dispatched messages are processed. +# native_transport_rate_limiting_enabled: false +# native_transport_max_requests_per_second: 1000000 + +# The address or interface to bind the native transport server to. +# +# Set rpc_address OR rpc_interface, not both. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +rpc_address: 0.0.0.0 + +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# rpc_interface: eth1 + +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +# rpc_interface_prefer_ipv6: false + +# RPC address to broadcast to drivers and other Cassandra nodes. This cannot +# be set to 0.0.0.0. If left blank, this will be set to the value of +# rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must +# be set. +broadcast_rpc_address: 172.18.0.2 + +# enable or disable keepalive on rpc/native connections +rpc_keepalive: true + +# Uncomment to set socket buffer size for internode communication +# Note that when setting this, the buffer size is limited by net.core.wmem_max +# and when not setting it it is defined by net.ipv4.tcp_wmem +# See also: +# /proc/sys/net/core/wmem_max +# /proc/sys/net/core/rmem_max +# /proc/sys/net/ipv4/tcp_wmem +# /proc/sys/net/ipv4/tcp_wmem +# and 'man tcp' +# Min unit: B +# internode_socket_send_buffer_size: + +# Uncomment to set socket buffer size for internode communication +# Note that when setting this, the buffer size is limited by net.core.wmem_max +# and when not setting it it is defined by net.ipv4.tcp_wmem +# Min unit: B +# internode_socket_receive_buffer_size: + +# Set to true to have Cassandra create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Cassandra won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +auto_snapshot: true + +# Adds a time-to-live (TTL) to auto snapshots generated by table +# truncation or drop (when enabled). +# After the TTL is elapsed, the snapshot is automatically cleared. +# By default, auto snapshots *do not* have TTL, uncomment the property below +# to enable TTL on auto snapshots. +# Accepted units: d (days), h (hours) or m (minutes) +# auto_snapshot_ttl: 30d + +# The act of creating or clearing a snapshot involves creating or removing +# potentially tens of thousands of links, which can cause significant performance +# impact, especially on consumer grade SSDs. A non-zero value here can +# be used to throttle these links to avoid negative performance impact of +# taking and clearing snapshots +snapshot_links_per_second: 0 + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# +# - a smaller granularity means more index entries are generated +# and looking up rows withing the partition by collation column +# is faster +# - but, Cassandra will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +# Min unit: KiB +column_index_size: 64KiB + +# Per sstable indexed key cache entries (the collation index in memory +# mentioned above) exceeding this size will not be held on heap. +# This means that only partition information is held on heap and the +# index entries are read from disk. +# +# Note that this size refers to the size of the +# serialized index information and not the size of the partition. +# Min unit: KiB +column_index_cache_size: 2KiB + +# Number of simultaneous compactions to allow, NOT including +# validation "compactions" for anti-entropy repair. Simultaneous +# compactions can help preserve read performance in a mixed read/write +# workload, by mitigating the tendency of small sstables to accumulate +# during a single long running compactions. The default is usually +# fine and if you experience problems with compaction running too +# slowly or too fast, you should look at +# compaction_throughput first. +# +# concurrent_compactors defaults to the smaller of (number of disks, +# number of cores), with a minimum of 2 and a maximum of 8. +# +# If your data directories are backed by SSD, you should increase this +# to the number of cores. +# concurrent_compactors: 1 + +# Number of simultaneous repair validations to allow. If not set or set to +# a value less than 1, it defaults to the value of concurrent_compactors. +# To set a value greeater than concurrent_compactors at startup, the system +# property cassandra.allow_unlimited_concurrent_validations must be set to +# true. To dynamically resize to a value > concurrent_compactors on a running +# node, first call the bypassConcurrentValidatorsLimit method on the +# org.apache.cassandra.db:type=StorageService mbean +# concurrent_validations: 0 + +# Number of simultaneous materialized view builder tasks to allow. +concurrent_materialized_view_builders: 1 + +# Throttles compaction to the given total throughput across the entire +# system. The faster you insert data, the faster you need to compact in +# order to keep the sstable count down, but in general, setting this to +# 16 to 32 times the rate you are inserting data is more than sufficient. +# Setting this to 0 disables throttling. Note that this accounts for all types +# of compaction, including validation compaction (building Merkle trees +# for repairs). +compaction_throughput: 64MiB/s + +# When compacting, the replacement sstable(s) can be opened before they +# are completely written, and used in place of the prior sstables for +# any range that has been written. This helps to smoothly transfer reads +# between the sstables, reducing page cache churn and keeping hot rows hot +# Set sstable_preemptive_open_interval to null for disabled which is equivalent to +# sstable_preemptive_open_interval_in_mb being negative +# Min unit: MiB +sstable_preemptive_open_interval: 50MiB + +# Starting from 4.1 sstables support UUID based generation identifiers. They are disabled by default +# because once enabled, there is no easy way to downgrade. When the node is restarted with this option +# set to true, each newly created sstable will have a UUID based generation identifier and such files are +# not readable by previous Cassandra versions. At some point, this option will become true by default +# and eventually get removed from the configuration. +uuid_sstable_identifiers_enabled: false + +# When enabled, permits Cassandra to zero-copy stream entire eligible +# SSTables between nodes, including every component. +# This speeds up the network transfer significantly subject to +# throttling specified by entire_sstable_stream_throughput_outbound, +# and entire_sstable_inter_dc_stream_throughput_outbound +# for inter-DC transfers. +# Enabling this will reduce the GC pressure on sending and receiving node. +# When unset, the default is enabled. While this feature tries to keep the +# disks balanced, it cannot guarantee it. This feature will be automatically +# disabled if internode encryption is enabled. +# stream_entire_sstables: true + +# Throttles entire SSTable outbound streaming file transfers on +# this node to the given total throughput in Mbps. +# Setting this value to 0 it disables throttling. +# When unset, the default is 200 Mbps or 24 MiB/s. +# entire_sstable_stream_throughput_outbound: 24MiB/s + +# Throttles entire SSTable file streaming between datacenters. +# Setting this value to 0 disables throttling for entire SSTable inter-DC file streaming. +# When unset, the default is 200 Mbps or 24 MiB/s. +# entire_sstable_inter_dc_stream_throughput_outbound: 24MiB/s + +# Throttles all outbound streaming file transfers on this node to the +# given total throughput in Mbps. This is necessary because Cassandra does +# mostly sequential IO when streaming data during bootstrap or repair, which +# can lead to saturating the network connection and degrading rpc performance. +# When unset, the default is 200 Mbps or 24 MiB/s. +# stream_throughput_outbound: 24MiB/s + +# Throttles all streaming file transfer between the datacenters, +# this setting allows users to throttle inter dc stream throughput in addition +# to throttling all network stream traffic as configured with +# stream_throughput_outbound_megabits_per_sec +# When unset, the default is 200 Mbps or 24 MiB/s. +# inter_dc_stream_throughput_outbound: 24MiB/s + +# Server side timeouts for requests. The server will return a timeout exception +# to the client if it can't complete an operation within the corresponding +# timeout. Those settings are a protection against: +# 1) having client wait on an operation that might never terminate due to some +# failures. +# 2) operations that use too much CPU/read too much data (leading to memory build +# up) by putting a limit to how long an operation will execute. +# For this reason, you should avoid putting these settings too high. In other words, +# if you are timing out requests because of underlying resource constraints then +# increasing the timeout will just cause more problems. Of course putting them too +# low is equally ill-advised since clients could get timeouts even for successful +# operations just because the timeout setting is too tight. + +# How long the coordinator should wait for read operations to complete. +# Lowest acceptable value is 10 ms. +# Min unit: ms +read_request_timeout: 5000ms +# How long the coordinator should wait for seq or index scans to complete. +# Lowest acceptable value is 10 ms. +# Min unit: ms +range_request_timeout: 10000ms +# How long the coordinator should wait for writes to complete. +# Lowest acceptable value is 10 ms. +# Min unit: ms +write_request_timeout: 2000ms +# How long the coordinator should wait for counter writes to complete. +# Lowest acceptable value is 10 ms. +# Min unit: ms +counter_write_request_timeout: 5000ms +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row. +# Lowest acceptable value is 10 ms. +# Min unit: ms +cas_contention_timeout: 1000ms +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +# Lowest acceptable value is 10 ms. +# Min unit: ms +truncate_request_timeout: 60000ms +# The default timeout for other, miscellaneous operations. +# Lowest acceptable value is 10 ms. +# Min unit: ms +request_timeout: 10000ms + +# Defensive settings for protecting Cassandra from true network partitions. +# See (CASSANDRA-14358) for details. +# +# The amount of time to wait for internode tcp connections to establish. +# Min unit: ms +# internode_tcp_connect_timeout: 2000ms +# +# The amount of time unacknowledged data is allowed on a connection before we throw out the connection +# Note this is only supported on Linux + epoll, and it appears to behave oddly above a setting of 30000 +# (it takes much longer than 30s) as of Linux 4.12. If you want something that high set this to 0 +# which picks up the OS default and configure the net.ipv4.tcp_retries2 sysctl to be ~8. +# Min unit: ms +# internode_tcp_user_timeout: 30000ms + +# The amount of time unacknowledged data is allowed on a streaming connection. +# The default is 5 minutes. Increase it or set it to 0 in order to increase the timeout. +# Min unit: ms +# internode_streaming_tcp_user_timeout: 300000ms + +# Global, per-endpoint and per-connection limits imposed on messages queued for delivery to other nodes +# and waiting to be processed on arrival from other nodes in the cluster. These limits are applied to the on-wire +# size of the message being sent or received. +# +# The basic per-link limit is consumed in isolation before any endpoint or global limit is imposed. +# Each node-pair has three links: urgent, small and large. So any given node may have a maximum of +# N*3*(internode_application_send_queue_capacity+internode_application_receive_queue_capacity) +# messages queued without any coordination between them although in practice, with token-aware routing, only RF*tokens +# nodes should need to communicate with significant bandwidth. +# +# The per-endpoint limit is imposed on all messages exceeding the per-link limit, simultaneously with the global limit, +# on all links to or from a single node in the cluster. +# The global limit is imposed on all messages exceeding the per-link limit, simultaneously with the per-endpoint limit, +# on all links to or from any node in the cluster. +# +# Min unit: B +# internode_application_send_queue_capacity: 4MiB +# internode_application_send_queue_reserve_endpoint_capacity: 128MiB +# internode_application_send_queue_reserve_global_capacity: 512MiB +# internode_application_receive_queue_capacity: 4MiB +# internode_application_receive_queue_reserve_endpoint_capacity: 128MiB +# internode_application_receive_queue_reserve_global_capacity: 512MiB + + +# How long before a node logs slow queries. Select queries that take longer than +# this timeout to execute, will generate an aggregated log message, so that slow queries +# can be identified. Set this value to zero to disable slow query logging. +# Min unit: ms +slow_query_log_timeout: 500ms + +# Enable operation timeout information exchange between nodes to accurately +# measure request timeouts. If disabled, replicas will assume that requests +# were forwarded to them instantly by the coordinator, which means that +# under overload conditions we will waste that much extra time processing +# already-timed-out requests. +# +# Warning: It is generally assumed that users have setup NTP on their clusters, and that clocks are modestly in sync, +# since this is a requirement for general correctness of last write wins. +# internode_timeout: true + +# Set period for idle state control messages for earlier detection of failed streams +# This node will send a keep-alive message periodically on the streaming's control channel. +# This ensures that any eventual SocketTimeoutException will occur within 2 keep-alive cycles +# If the node cannot send, or timeouts sending, the keep-alive message on the netty control channel +# the stream session is closed. +# Default value is 300s (5 minutes), which means stalled streams +# are detected within 10 minutes +# Specify 0 to disable. +# Min unit: s +# streaming_keep_alive_period: 300s + +# Limit number of connections per host for streaming +# Increase this when you notice that joins are CPU-bound rather that network +# bound (for example a few nodes with big files). +# streaming_connections_per_host: 1 + +# Settings for stream stats tracking; used by system_views.streaming table +# How long before a stream is evicted from tracking; this impacts both historic and currently running +# streams. +# streaming_state_expires: 3d +# How much memory may be used for tracking before evicting session from tracking; once crossed +# historic and currently running streams maybe impacted. +# streaming_state_size: 40MiB +# Enable/Disable tracking of streaming stats +# streaming_stats_enabled: true + +# Allows denying configurable access (rw/rr) to operations on configured ks, table, and partitions, intended for use by +# operators to manage cluster health vs application access. See CASSANDRA-12106 and CEP-13 for more details. +# partition_denylist_enabled: false + +# denylist_writes_enabled: true +# denylist_reads_enabled: true +# denylist_range_reads_enabled: true + +# The interval at which keys in the cache for denylisting will "expire" and async refresh from the backing DB. +# Note: this serves only as a fail-safe, as the usage pattern is expected to be "mutate state, refresh cache" on any +# changes to the underlying denylist entries. See documentation for details. +# Min unit: s +# denylist_refresh: 600s + +# In the event of errors on attempting to load the denylist cache, retry on this interval. +# Min unit: s +# denylist_initial_load_retry: 5s + +# We cap the number of denylisted keys allowed per table to keep things from growing unbounded. Nodes will warn above +# this limit while allowing new denylisted keys to be inserted. Denied keys are loaded in natural query / clustering +# ordering by partition key in case of overflow. +# denylist_max_keys_per_table: 1000 + +# We cap the total number of denylisted keys allowed in the cluster to keep things from growing unbounded. +# Nodes will warn on initial cache load that there are too many keys and be direct the operator to trim down excess +# entries to within the configured limits. +# denylist_max_keys_total: 10000 + +# Since the denylist in many ways serves to protect the health of the cluster from partitions operators have identified +# as being in a bad state, we usually want more robustness than just CL.ONE on operations to/from these tables to +# ensure that these safeguards are in place. That said, we allow users to configure this if they're so inclined. +# denylist_consistency_level: QUORUM + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# endpoint_snitch -- Set this to a class that implements +# IEndpointSnitch. The snitch has two functions: +# +# - it teaches Cassandra enough about your network topology to route +# requests efficiently +# - it allows Cassandra to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Cassandra will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# CASSANDRA WILL NOT ALLOW YOU TO SWITCH TO AN INCOMPATIBLE SNITCH +# ONCE DATA IS INSERTED INTO THE CLUSTER. This would cause data loss. +# This means that if you start with the default SimpleSnitch, which +# locates every node on "rack1" in "datacenter1", your only options +# if you need to add another datacenter are GossipingPropertyFileSnitch +# (and the older PFS). From there, if you want to migrate to an +# incompatible snitch like Ec2Snitch you can do it by adding new nodes +# under Ec2Snitch (which will locate them in a new "datacenter") and +# decommissioning the old ones. +# +# Out of the box, Cassandra provides: +# +# SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# +# GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# +# PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# +# Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# +# Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Cassandra will switch to the private IP after +# establishing a connection.) +# +# RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: SimpleSnitch + +# controls how often to perform the more expensive part of host score +# calculation +# Min unit: ms +dynamic_snitch_update_interval: 100ms +# controls how often to reset all host scores, allowing a bad host to +# possibly recover +# Min unit: ms +dynamic_snitch_reset_interval: 600000ms +# if set greater than zero, this will allow +# 'pinning' of replicas to hosts in order to increase cache capacity. +# The badness threshold will control how much worse the pinned host has to be +# before the dynamic snitch will prefer other replicas over it. This is +# expressed as a double which represents a percentage. Thus, a value of +# 0.2 means Cassandra would continue to prefer the static snitch values +# until the pinned host was 20% worse than the fastest. +dynamic_snitch_badness_threshold: 1.0 + +# Configure server-to-server internode encryption +# +# JVM and netty defaults for supported SSL socket protocols and cipher suites can +# be replaced using custom encryption options. This is not recommended +# unless you have policies in place that dictate certain settings, or +# need to disable vulnerable ciphers or protocols in case the JVM cannot +# be updated. +# +# FIPS compliant settings can be configured at JVM level and should not +# involve changing encryption settings here: +# https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/FIPS.html +# +# **NOTE** this default configuration is an insecure configuration. If you need to +# enable server-to-server encryption generate server keystores (and truststores for mutual +# authentication) per: +# http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore +# Then perform the following configuration changes: +# +# Step 1: Set internode_encryption= and explicitly set optional=true. Restart all nodes +# +# Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual +# auth set require_client_auth=true. Restart all nodes +server_encryption_options: + # On outbound connections, determine which type of peers to securely connect to. + # The available options are : + # none : Do not encrypt outgoing connections + # dc : Encrypt connections to peers in other datacenters but not within datacenters + # rack : Encrypt connections to peers in other racks but not within racks + # all : Always use encrypted connections + internode_encryption: none + # When set to true, encrypted and unencrypted connections are allowed on the storage_port + # This should _only be true_ while in unencrypted or transitional operation + # optional defaults to true if internode_encryption is none + # optional: true + # If enabled, will open up an encrypted listening socket on ssl_storage_port. Should only be used + # during upgrade to 4.0; otherwise, set to false. + legacy_ssl_storage_port_enabled: false + # Set to a valid keystore if internode_encryption is dc, rack or all + keystore: conf/.keystore + keystore_password: cassandra + # Configure the way Cassandra creates SSL contexts. + # To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory + # ssl_context_factory: + # # Must be an instance of org.apache.cassandra.security.ISslContextFactory + # class_name: org.apache.cassandra.security.DefaultSslContextFactory + # Verify peer server certificates + require_client_auth: false + # Set to a valid trustore if require_client_auth is true + truststore: conf/.truststore + truststore_password: cassandra + # Verify that the host name in the certificate matches the connected host + require_endpoint_verification: false + # More advanced defaults: + # protocol: TLS + # store_type: JKS + # cipher_suites: [ + # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, + # TLS_RSA_WITH_AES_256_CBC_SHA + # ] + +# Configure client-to-server encryption. +# +# **NOTE** this default configuration is an insecure configuration. If you need to +# enable client-to-server encryption generate server keystores (and truststores for mutual +# authentication) per: +# http://download.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore +# Then perform the following configuration changes: +# +# Step 1: Set enabled=true and explicitly set optional=true. Restart all nodes +# +# Step 2: Set optional=false (or remove it) and if you generated truststores and want to use mutual +# auth set require_client_auth=true. Restart all nodes +client_encryption_options: + # Enable client-to-server encryption + enabled: false + # When set to true, encrypted and unencrypted connections are allowed on the native_transport_port + # This should _only be true_ while in unencrypted or transitional operation + # optional defaults to true when enabled is false, and false when enabled is true. + # optional: true + # Set keystore and keystore_password to valid keystores if enabled is true + keystore: conf/.keystore + keystore_password: cassandra + # Configure the way Cassandra creates SSL contexts. + # To use PEM-based key material, see org.apache.cassandra.security.PEMBasedSslContextFactory + # ssl_context_factory: + # # Must be an instance of org.apache.cassandra.security.ISslContextFactory + # class_name: org.apache.cassandra.security.DefaultSslContextFactory + # Verify client certificates + require_client_auth: false + # Set trustore and truststore_password if require_client_auth is true + # truststore: conf/.truststore + # truststore_password: cassandra + # More advanced defaults: + # protocol: TLS + # store_type: JKS + # cipher_suites: [ + # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + # TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA, + # TLS_RSA_WITH_AES_256_CBC_SHA + # ] + +# internode_compression controls whether traffic between nodes is +# compressed. +# Can be: +# +# all +# all traffic is compressed +# +# dc +# traffic between different datacenters is compressed +# +# none +# nothing is compressed. +internode_compression: dc + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +inter_dc_tcp_nodelay: false + +# TTL for different trace types used during logging of the repair process. +# Min unit: s +trace_type_query_ttl: 1d +# Min unit: s +trace_type_repair_ttl: 7d + +# If unset, all GC Pauses greater than gc_log_threshold will log at +# INFO level +# UDFs (user defined functions) are disabled by default. +# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code. +user_defined_functions_enabled: false + +# Enables scripted UDFs (JavaScript UDFs). +# Java UDFs are always enabled, if user_defined_functions_enabled is true. +# Enable this option to be able to use UDFs with "language javascript" or any custom JSR-223 provider. +# This option has no effect, if user_defined_functions_enabled is false. +scripted_user_defined_functions_enabled: false + +# Enables encrypting data at-rest (on disk). Different key providers can be plugged in, but the default reads from +# a JCE-style keystore. A single keystore can hold multiple keys, but the one referenced by +# the "key_alias" is the only key that will be used for encrypt opertaions; previously used keys +# can still (and should!) be in the keystore and will be used on decrypt operations +# (to handle the case of key rotation). +# +# It is strongly recommended to download and install Java Cryptography Extension (JCE) +# Unlimited Strength Jurisdiction Policy Files for your version of the JDK. +# (current link: http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html) +# +# Currently, only the following file types are supported for transparent data encryption, although +# more are coming in future cassandra releases: commitlog, hints +transparent_data_encryption_options: + enabled: false + chunk_length_kb: 64 + cipher: AES/CBC/PKCS5Padding + key_alias: testing:1 + # CBC IV length for AES needs to be 16 bytes (which is also the default size) + # iv_length: 16 + key_provider: + - class_name: org.apache.cassandra.security.JKSKeyProvider + parameters: + - keystore: conf/.keystore + keystore_password: cassandra + store_type: JCEKS + key_password: cassandra + + +##################### +# SAFETY THRESHOLDS # +##################### + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +tombstone_warn_threshold: 1000 +tombstone_failure_threshold: 100000 + +# Filtering and secondary index queries at read consistency levels above ONE/LOCAL_ONE use a +# mechanism called replica filtering protection to ensure that results from stale replicas do +# not violate consistency. (See CASSANDRA-8272 and CASSANDRA-15907 for more details.) This +# mechanism materializes replica results by partition on-heap at the coordinator. The more possibly +# stale results returned by the replicas, the more rows materialized during the query. +replica_filtering_protection: + # These thresholds exist to limit the damage severely out-of-date replicas can cause during these + # queries. They limit the number of rows from all replicas individual index and filtering queries + # can materialize on-heap to return correct results at the desired read consistency level. + # + # "cached_replica_rows_warn_threshold" is the per-query threshold at which a warning will be logged. + # "cached_replica_rows_fail_threshold" is the per-query threshold at which the query will fail. + # + # These thresholds may also be adjusted at runtime using the StorageService mbean. + # + # If the failure threshold is breached, it is likely that either the current page/fetch size + # is too large or one or more replicas is severely out-of-sync and in need of repair. + cached_rows_warn_threshold: 2000 + cached_rows_fail_threshold: 32000 + +# Log WARN on any multiple-partition batch size exceeding this value. 5KiB per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +# Min unit: KiB +batch_size_warn_threshold: 5KiB + +# Fail any multiple-partition batch exceeding this value. 50KiB (10x warn threshold) by default. +# Min unit: KiB +batch_size_fail_threshold: 50KiB + +# Log WARN on any batches not of type LOGGED than span across more partitions than this limit +unlogged_batch_across_partitions_warn_threshold: 10 + +# Log a warning when compacting partitions larger than this value +compaction_large_partition_warning_threshold: 100MiB + +# Log a warning when writing more tombstones than this value to a partition +compaction_tombstone_warning_threshold: 100000 + +# GC Pauses greater than 200 ms will be logged at INFO level +# This threshold can be adjusted to minimize logging if necessary +# Min unit: ms +# gc_log_threshold: 200ms + +# GC Pauses greater than gc_warn_threshold will be logged at WARN level +# Adjust the threshold based on your application throughput requirement. Setting to 0 +# will deactivate the feature. +# Min unit: ms +# gc_warn_threshold: 1000ms + +# Maximum size of any value in SSTables. Safety measure to detect SSTable corruption +# early. Any value size larger than this threshold will result into marking an SSTable +# as corrupted. This should be positive and less than 2GiB. +# Min unit: MiB +# max_value_size: 256MiB + +# ** Impact on keyspace creation ** +# If replication factor is not mentioned as part of keyspace creation, default_keyspace_rf would apply. +# Changing this configuration would only take effect for keyspaces created after the change, but does not impact +# existing keyspaces created prior to the change. +# ** Impact on keyspace alter ** +# When altering a keyspace from NetworkTopologyStrategy to SimpleStrategy, default_keyspace_rf is applied if rf is not +# explicitly mentioned. +# ** Impact on system keyspaces ** +# This would also apply for any system keyspaces that need replication factor. +# A further note about system keyspaces - system_traces and system_distributed keyspaces take RF of 2 or default, +# whichever is higher, and system_auth keyspace takes RF of 1 or default, whichever is higher. +# Suggested value for use in production: 3 +# default_keyspace_rf: 1 + +# Track a metric per keyspace indicating whether replication achieved the ideal consistency +# level for writes without timing out. This is different from the consistency level requested by +# each write which may be lower in order to facilitate availability. +# ideal_consistency_level: EACH_QUORUM + +# Automatically upgrade sstables after upgrade - if there is no ordinary compaction to do, the +# oldest non-upgraded sstable will get upgraded to the latest version +# automatic_sstable_upgrade: false +# Limit the number of concurrent sstable upgrades +# max_concurrent_automatic_sstable_upgrades: 1 + +# Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs +# on audit_logging for full details about the various configuration options. +audit_logging_options: + enabled: false + logger: + - class_name: BinAuditLogger + # audit_logs_dir: + # included_keyspaces: + # excluded_keyspaces: system, system_schema, system_virtual_schema + # included_categories: + # excluded_categories: + # included_users: + # excluded_users: + # roll_cycle: HOURLY + # block: true + # max_queue_weight: 268435456 # 256 MiB + # max_log_size: 17179869184 # 16 GiB + ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled: + # archive_command: + # max_archive_retries: 10 + + +# default options for full query logging - these can be overridden from command line when executing +# nodetool enablefullquerylog +# full_query_logging_options: + # log_dir: + # roll_cycle: HOURLY + # block: true + # max_queue_weight: 268435456 # 256 MiB + # max_log_size: 17179869184 # 16 GiB + ## archive command is "/path/to/script.sh %path" where %path is replaced with the file being rolled: + # archive_command: + ## note that enabling this allows anyone with JMX/nodetool access to run local shell commands as the user running cassandra + # allow_nodetool_archive_command: false + # max_archive_retries: 10 + +# validate tombstones on reads and compaction +# can be either "disabled", "warn" or "exception" +# corrupted_tombstone_strategy: disabled + +# Diagnostic Events # +# If enabled, diagnostic events can be helpful for troubleshooting operational issues. Emitted events contain details +# on internal state and temporal relationships across events, accessible by clients via JMX. +diagnostic_events_enabled: false + +# Use native transport TCP message coalescing. If on upgrade to 4.0 you found your throughput decreasing, and in +# particular you run an old kernel or have very fewer client connections, this option might be worth evaluating. +#native_transport_flush_in_batches_legacy: false + +# Enable tracking of repaired state of data during reads and comparison between replicas +# Mismatches between the repaired sets of replicas can be characterized as either confirmed +# or unconfirmed. In this context, unconfirmed indicates that the presence of pending repair +# sessions, unrepaired partition tombstones, or some other condition means that the disparity +# cannot be considered conclusive. Confirmed mismatches should be a trigger for investigation +# as they may be indicative of corruption or data loss. +# There are separate flags for range vs partition reads as single partition reads are only tracked +# when CL > 1 and a digest mismatch occurs. Currently, range queries don't use digests so if +# enabled for range reads, all range reads will include repaired data tracking. As this adds +# some overhead, operators may wish to disable it whilst still enabling it for partition reads +repaired_data_tracking_for_range_reads_enabled: false +repaired_data_tracking_for_partition_reads_enabled: false +# If false, only confirmed mismatches will be reported. If true, a separate metric for unconfirmed +# mismatches will also be recorded. This is to avoid potential signal:noise issues are unconfirmed +# mismatches are less actionable than confirmed ones. +report_unconfirmed_repaired_data_mismatches: false + +# Having many tables and/or keyspaces negatively affects performance of many operations in the +# cluster. When the number of tables/keyspaces in the cluster exceeds the following thresholds +# a client warning will be sent back to the user when creating a table or keyspace. +# As of cassandra 4.1, these properties are deprecated in favor of keyspaces_warn_threshold and tables_warn_threshold +# table_count_warn_threshold: 150 +# keyspace_count_warn_threshold: 40 + +# configure the read and write consistency levels for modifications to auth tables +# auth_read_consistency_level: LOCAL_QUORUM +# auth_write_consistency_level: EACH_QUORUM + +# Delays on auth resolution can lead to a thundering herd problem on reconnects; this option will enable +# warming of auth caches prior to node completing startup. See CASSANDRA-16958 +# auth_cache_warming_enabled: false + +######################### +# EXPERIMENTAL FEATURES # +######################### + +# Enables materialized view creation on this node. +# Materialized views are considered experimental and are not recommended for production use. +materialized_views_enabled: true + +# Enables SASI index creation on this node. +# SASI indexes are considered experimental and are not recommended for production use. +sasi_indexes_enabled: false + +# Enables creation of transiently replicated keyspaces on this node. +# Transient replication is experimental and is not recommended for production use. +transient_replication_enabled: false + +# Enables the used of 'ALTER ... DROP COMPACT STORAGE' statements on this node. +# 'ALTER ... DROP COMPACT STORAGE' is considered experimental and is not recommended for production use. +drop_compact_storage_enabled: false + +# Whether or not USE is allowed. This is enabled by default to avoid failure on upgrade. +#use_statements_enabled: true + +# When the client triggers a protocol exception or unknown issue (Cassandra bug) we increment +# a client metric showing this; this logic will exclude specific subnets from updating these +# metrics +#client_error_reporting_exclusions: +# subnets: +# - 127.0.0.1 +# - 127.0.0.0/31 + +# Enables read thresholds (warn/fail) across all replicas for reporting back to the client. +# See: CASSANDRA-16850 +# read_thresholds_enabled: false # scheduled to be set true in 4.2 +# When read_thresholds_enabled: true, this tracks the materialized size of a query on the +# coordinator. If coordinator_read_size_warn_threshold is defined, this will emit a warning +# to clients with details on what query triggered this as well as the size of the result set; if +# coordinator_read_size_fail_threshold is defined, this will fail the query after it +# has exceeded this threshold, returning a read error to the user. +# coordinator_read_size_warn_threshold: +# coordinator_read_size_fail_threshold: +# When read_thresholds_enabled: true, this tracks the size of the local read (as defined by +# heap size), and will warn/fail based off these thresholds; undefined disables these checks. +# local_read_size_warn_threshold: +# local_read_size_fail_threshold: +# When read_thresholds_enabled: true, this tracks the expected memory size of the RowIndexEntry +# and will warn/fail based off these thresholds; undefined disables these checks +# row_index_read_size_warn_threshold: +# row_index_read_size_fail_threshold: + +# Guardrail to warn or fail when creating more user keyspaces than threshold. +# The two thresholds default to -1 to disable. +# keyspaces_warn_threshold: -1 +# keyspaces_fail_threshold: -1 +# Guardrail to warn or fail when creating more user tables than threshold. +# The two thresholds default to -1 to disable. +# tables_warn_threshold: -1 +# tables_fail_threshold: -1 +# Guardrail to enable or disable the ability to create uncompressed tables +# uncompressed_tables_enabled: true +# Guardrail to warn or fail when creating/altering a table with more columns per table than threshold. +# The two thresholds default to -1 to disable. +# columns_per_table_warn_threshold: -1 +# columns_per_table_fail_threshold: -1 +# Guardrail to warn or fail when creating more secondary indexes per table than threshold. +# The two thresholds default to -1 to disable. +# secondary_indexes_per_table_warn_threshold: -1 +# secondary_indexes_per_table_fail_threshold: -1 +# Guardrail to enable or disable the creation of secondary indexes +# secondary_indexes_enabled: true +# Guardrail to warn or fail when creating more materialized views per table than threshold. +# The two thresholds default to -1 to disable. +# materialized_views_per_table_warn_threshold: -1 +# materialized_views_per_table_fail_threshold: -1 +# Guardrail to warn about, ignore or reject properties when creating tables. By default all properties are allowed. +# table_properties_warned: [] +# table_properties_ignored: [] +# table_properties_disallowed: [] +# Guardrail to allow/disallow user-provided timestamps. Defaults to true. +# user_timestamps_enabled: true +# Guardrail to allow/disallow GROUP BY functionality. +# group_by_enabled: true +# Guardrail to allow/disallow TRUNCATE and DROP TABLE statements +# drop_truncate_table_enabled: true +# Guardrail to warn or fail when using a page size greater than threshold. +# The two thresholds default to -1 to disable. +# page_size_warn_threshold: -1 +# page_size_fail_threshold: -1 +# Guardrail to allow/disallow list operations that require read before write, i.e. setting list element by index and +# removing list elements by either index or value. Defaults to true. +# read_before_write_list_operations_enabled: true +# Guardrail to warn or fail when querying with an IN restriction selecting more partition keys than threshold. +# The two thresholds default to -1 to disable. +# partition_keys_in_select_warn_threshold: -1 +# partition_keys_in_select_fail_threshold: -1 +# Guardrail to warn or fail when an IN query creates a cartesian product with a size exceeding threshold, +# eg. "a in (1,2,...10) and b in (1,2...10)" results in cartesian product of 100. +# The two thresholds default to -1 to disable. +# in_select_cartesian_product_warn_threshold: -1 +# in_select_cartesian_product_fail_threshold: -1 +# Guardrail to warn about or reject read consistency levels. By default, all consistency levels are allowed. +# read_consistency_levels_warned: [] +# read_consistency_levels_disallowed: [] +# Guardrail to warn about or reject write consistency levels. By default, all consistency levels are allowed. +# write_consistency_levels_warned: [] +# write_consistency_levels_disallowed: [] +# Guardrail to warn or fail when encountering larger size of collection data than threshold. +# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case +# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to +# prevent read-before-write. The guardrail is also checked at sstable write time to detect large non-frozen collections, +# although in that case exceeding the fail threshold will only log an error message, without interrupting the operation. +# The two thresholds default to null to disable. +# Min unit: B +# collection_size_warn_threshold: +# Min unit: B +# collection_size_fail_threshold: +# Guardrail to warn or fail when encountering more elements in collection than threshold. +# At query time this guardrail is applied only to the collection fragment that is being writen, even though in the case +# of non-frozen collections there could be unaccounted parts of the collection on the sstables. This is done this way to +# prevent read-before-write. The guardrail is also checked at sstable write time to detect large non-frozen collections, +# although in that case exceeding the fail threshold will only log an error message, without interrupting the operation. +# The two thresholds default to -1 to disable. +# items_per_collection_warn_threshold: -1 +# items_per_collection_fail_threshold: -1 +# Guardrail to allow/disallow querying with ALLOW FILTERING. Defaults to true. +# allow_filtering_enabled: true +# Guardrail to warn or fail when creating a user-defined-type with more fields in than threshold. +# Default -1 to disable. +# fields_per_udt_warn_threshold: -1 +# fields_per_udt_fail_threshold: -1 +# Guardrail to warn or fail when local data disk usage percentage exceeds threshold. Valid values are in [1, 100]. +# This is only used for the disks storing data directories, so it won't count any separate disks used for storing +# the commitlog, hints nor saved caches. The disk usage is the ratio between the amount of space used by the data +# directories and the addition of that same space and the remaining free space on disk. The main purpose of this +# guardrail is rejecting user writes when the disks are over the defined usage percentage, so the writes done by +# background processes such as compaction and streaming don't fail due to a full disk. The limits should be defined +# accordingly to the expected data growth due to those background processes, so for example a compaction strategy +# doubling the size of the data would require to keep the disk usage under 50%. +# The two thresholds default to -1 to disable. +# data_disk_usage_percentage_warn_threshold: -1 +# data_disk_usage_percentage_fail_threshold: -1 +# Allows defining the max disk size of the data directories when calculating thresholds for +# disk_usage_percentage_warn_threshold and disk_usage_percentage_fail_threshold, so if this is greater than zero they +# become percentages of a fixed size on disk instead of percentages of the physically available disk size. This should +# be useful when we have a large disk and we only want to use a part of it for Cassandra's data directories. +# Valid values are in [1, max available disk size of all data directories]. +# Defaults to null to disable and use the physically available disk size of data directories during calculations. +# Min unit: B +# data_disk_usage_max_disk_size: +# Guardrail to warn or fail when the minimum replication factor is lesser than threshold. +# This would also apply to system keyspaces. +# Suggested value for use in production: 2 or higher +# minimum_replication_factor_warn_threshold: -1 +# minimum_replication_factor_fail_threshold: -1 + +# Startup Checks are executed as part of Cassandra startup process, not all of them +# are configurable (so you can disable them) but these which are enumerated bellow. +# Uncomment the startup checks and configure them appropriately to cover your needs. +# +#startup_checks: +# Verifies correct ownership of attached locations on disk at startup. See CASSANDRA-16879 for more details. +# check_filesystem_ownership: +# enabled: false +# ownership_token: "sometoken" # (overriden by "CassandraOwnershipToken" system property) +# ownership_filename: ".cassandra_fs_ownership" # (overriden by "cassandra.fs_ownership_filename") +# Prevents a node from starting if snitch's data center differs from previous data center. +# check_dc: +# enabled: true # (overriden by cassandra.ignore_dc system property) +# Prevents a node from starting if snitch's rack differs from previous rack. +# check_rack: +# enabled: true # (overriden by cassandra.ignore_rack system property) +# Enable this property to fail startup if the node is down for longer than gc_grace_seconds, to potentially +# prevent data resurrection on tables with deletes. By default, this will run against all keyspaces and tables +# except the ones specified on excluded_keyspaces and excluded_tables. +# check_data_resurrection: +# enabled: false +# file where Cassandra periodically writes the last time it was known to run +# heartbeat_file: /var/lib/cassandra/data/cassandra-heartbeat +# excluded_keyspaces: # comma separated list of keyspaces to exclude from the check +# excluded_tables: # comma separated list of keyspace.table pairs to exclude from the check diff --git a/metadata-ingestion/tests/integration/cassandra/setup/init_keyspaces.cql b/metadata-ingestion/tests/integration/cassandra/setup/init_keyspaces.cql new file mode 100644 index 00000000000000..81e919461a55bc --- /dev/null +++ b/metadata-ingestion/tests/integration/cassandra/setup/init_keyspaces.cql @@ -0,0 +1,131 @@ + +CREATE KEYSPACE cass_test_1 WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1}; +CREATE KEYSPACE cass_test_2 WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1}; + +CREATE TABLE cass_test_1.information ( + person_id int PRIMARY KEY, + last_updated timestamp, + details text +); + +CREATE TABLE cass_test_1.people ( + person_id int PRIMARY KEY, + name text, + email text +); + +CREATE TABLE cass_test_2.tasks ( + task_id int PRIMARY KEY, + last_updated timestamp, + details text, + status text +); + +CREATE MATERIALIZED VIEW cass_test_2.task_status AS +SELECT + task_id, + status +FROM cass_test_2.tasks +WHERE status IS NOT NULL AND task_id IS NOT NULL +PRIMARY KEY (task_id, status); + +-- Create Keyspace with comments +CREATE KEYSPACE IF NOT EXISTS example_keyspace +WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}; + +-- Use Keyspace +USE example_keyspace; + +-- Table with non-counter column types +CREATE TABLE IF NOT EXISTS all_data_types ( + id uuid PRIMARY KEY, + ascii_column ascii, + bigint_column bigint, + blob_column blob, + boolean_column boolean, + date_column date, + decimal_column decimal, + double_column double, + float_column float, + inet_column inet, + int_column int, + list_column list, + map_column map, + set_column set, + smallint_column smallint, + text_column text, + time_column time, + timestamp_column timestamp, + timeuuid_column timeuuid, + tinyint_column tinyint, + tuple_column tuple, + uuid_column uuid, + varchar_column varchar, + varint_column varint, + frozen_map_column frozen>, + frozen_list_column frozen>, + frozen_set_column frozen> +) WITH COMMENT = 'Table containing all supported Cassandra data types, excluding counters'; + +-- Separate table for counters +CREATE TABLE IF NOT EXISTS counter_table ( + id uuid PRIMARY KEY, + counter_column counter +) WITH COMMENT = 'Separate table containing only counter column'; + +-- Sample view +CREATE MATERIALIZED VIEW IF NOT EXISTS example_view_1 AS + SELECT id, ascii_column, bigint_column + FROM all_data_types + WHERE id IS NOT NULL AND ascii_column IS NOT NULL + PRIMARY KEY (id, ascii_column) WITH COMMENT = 'Example view definition with id and ascii_column'; + +CREATE MATERIALIZED VIEW IF NOT EXISTS example_view_2 AS + SELECT id, ascii_column, float_column + FROM all_data_types + WHERE id IS NOT NULL AND ascii_column IS NOT NULL + PRIMARY KEY (id, ascii_column) WITH COMMENT = 'Example view definition with id and ascii_column'; + +-- Table created for profilling +CREATE TABLE IF NOT EXISTS shopping_cart ( +userid text PRIMARY KEY, +item_count int, +last_update_timestamp timestamp +); + +-- Insert some data +INSERT INTO shopping_cart +(userid, item_count, last_update_timestamp) +VALUES ('9876', 2, '2024-11-01T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, item_count, last_update_timestamp) +VALUES ('1234', 5, '2024-11-02T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, item_count, last_update_timestamp) +VALUES ('1235', 100, '2024-11-03T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, item_count, last_update_timestamp) +VALUES ('1236', 50, '2024-11-04T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, item_count, last_update_timestamp) +VALUES ('1237', 75, '2024-11-05T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, last_update_timestamp) +VALUES ('1238', '2024-11-06T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, last_update_timestamp) +VALUES ('1239', '2024-11-07T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, last_update_timestamp) +VALUES ('1240', '2024-11-08T00:00:00.000+0000'); + +INSERT INTO shopping_cart +(userid, last_update_timestamp) +VALUES ('1241', '2024-11-09T00:00:00.000+0000'); \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/cassandra/test_cassandra.py b/metadata-ingestion/tests/integration/cassandra/test_cassandra.py new file mode 100644 index 00000000000000..d561308aaad20e --- /dev/null +++ b/metadata-ingestion/tests/integration/cassandra/test_cassandra.py @@ -0,0 +1,53 @@ +import logging +import time + +import pytest + +from datahub.ingestion.run.pipeline import Pipeline +from tests.test_helpers import mce_helpers +from tests.test_helpers.docker_helpers import wait_for_port + +logger = logging.getLogger(__name__) + + +@pytest.mark.integration +def test_cassandra_ingest(docker_compose_runner, pytestconfig, tmp_path): + test_resources_dir = pytestconfig.rootpath / "tests/integration/cassandra" + + with docker_compose_runner( + test_resources_dir / "docker-compose.yml", "cassandra" + ) as docker_services: + wait_for_port(docker_services, "test-cassandra", 9042) + + time.sleep(5) + # Run the metadata ingestion pipeline. + logger.info("Starting the ingestion test...") + pipeline_default_platform_instance = Pipeline.create( + { + "run_id": "cassandra-test", + "source": { + "type": "cassandra", + "config": { + "contact_point": "localhost", + "port": 9042, + "profiling": {"enabled": True}, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/cassandra_mcps.json", + }, + }, + } + ) + pipeline_default_platform_instance.run() + pipeline_default_platform_instance.raise_from_status() + + # Verify the output. + logger.info("Verifying output.") + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/cassandra_mcps.json", + golden_path=test_resources_dir / "cassandra_mcps_golden.json", + ) diff --git a/metadata-ingestion/tests/unit/test_cassandra_source.py b/metadata-ingestion/tests/unit/test_cassandra_source.py new file mode 100644 index 00000000000000..a4ca3a0a9ef3f6 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_cassandra_source.py @@ -0,0 +1,81 @@ +import json +import logging +import re +from typing import Any, Dict, List, Tuple + +import pytest + +from datahub.ingestion.source.cassandra.cassandra import CassandraToSchemaFieldConverter +from datahub.ingestion.source.cassandra.cassandra_api import CassandraColumn +from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField + +logger = logging.getLogger(__name__) + + +def assert_field_paths_are_unique(fields: List[SchemaField]) -> None: + fields_paths = [f.fieldPath for f in fields if re.match(".*[^]]$", f.fieldPath)] + + if fields_paths: + assert len(fields_paths) == len(set(fields_paths)) + + +def assert_field_paths_match( + fields: List[SchemaField], expected_field_paths: List[str] +) -> None: + logger.debug('FieldPaths=\n"' + '",\n"'.join(f.fieldPath for f in fields) + '"') + assert len(fields) == len(expected_field_paths) + for f, efp in zip(fields, expected_field_paths): + assert f.fieldPath == efp + assert_field_paths_are_unique(fields) + + +# TODO: cover one for every item on https://cassandra.apache.org/doc/stable/cassandra/cql/types.html (version 4.1) +schema_test_cases: Dict[str, Tuple[str, List[str]]] = { + "all_types_on_4.1": ( + """{ + "column_infos": [ + {"keyspace_name": "playground", "table_name": "people", "column_name": "birthday", "clustering_order": "none", "column_name_bytes": null, "kind": "regular", "position": -1, "type": "timestamp"}, + {"keyspace_name": "playground", "table_name": "people", "column_name": "email", "clustering_order": "none", "column_name_bytes": null, "kind": "partition_key", "position": 0, "type": "text"}, + {"keyspace_name": "playground", "table_name": "people", "column_name": "name", "clustering_order": "none", "column_name_bytes": null, "kind": "regular", "position": -1, "type": "text"} + ] + }""", + [ + "birthday", + "email", + "name", + ], + ) +} + + +@pytest.mark.parametrize( + "schema, expected_field_paths", + schema_test_cases.values(), + ids=schema_test_cases.keys(), +) +def test_cassandra_schema_conversion( + schema: str, expected_field_paths: List[str] +) -> None: + + schema_dict: Dict[str, List[Any]] = json.loads(schema) + column_infos: List = schema_dict["column_infos"] + + column_list: List[CassandraColumn] = [ + CassandraColumn( + keyspace_name=row["keyspace_name"], + table_name=row["table_name"], + column_name=row["column_name"], + clustering_order=row["clustering_order"], + kind=row["kind"], + position=row["position"], + type=row["type"], + ) + for row in column_infos + ] + actual_fields = list(CassandraToSchemaFieldConverter.get_schema_fields(column_list)) + assert_field_paths_match(actual_fields, expected_field_paths) + + +def test_no_properties_in_mappings_schema() -> None: + fields = list(CassandraToSchemaFieldConverter.get_schema_fields([])) + assert fields == [] diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml index f480ec862bc4e2..1625df4a99540d 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml @@ -717,3 +717,13 @@ displayName: Dremio type: QUERY_ENGINE logoUrl: "/assets/platforms/dremiologo.png" +- entityUrn: urn:li:dataPlatform:cassandra + entityType: dataPlatform + aspectName: dataPlatformInfo + changeType: UPSERT + aspect: + datasetNameDelimiter: "." + name: cassandra + displayName: Cassandra + type: KEY_VALUE_STORE + logoUrl: "/assets/platforms/cassandralogo.png" From 35c2a91e3a8f641f785390f06469e5cf80f854ff Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:48:47 -0600 Subject: [PATCH 008/174] search(entity-type): make searchable EntityTypeKey (#11868) --- .../src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl | 1 + 1 file changed, 1 insertion(+) diff --git a/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl index d857c7ff611e30..6a87c3d63cbd94 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/entitytype/EntityTypeKey.pdl @@ -7,5 +7,6 @@ record EntityTypeKey { /** * A unique id for an entity type. Usually this will be a unique namespace + entity name. */ + @Searchable = {} id: string } From 576ab6a19c901f00b7a4c408e3e9bd1ecc5d7913 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:39:21 -0600 Subject: [PATCH 009/174] docs(search): example tag AND condition (#11870) --- docs/how/search.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/how/search.md b/docs/how/search.md index 2274fe7c09240d..652338f8f581a9 100644 --- a/docs/how/search.md +++ b/docs/how/search.md @@ -359,6 +359,33 @@ queryConfigurations: boost_mode: multiply ``` +Similar example to boost with `primary` AND `gold` instead of the previous OR condition. + +```yaml +queryConfigurations: + - queryRegex: .* + + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + + functionScore: + functions: + + - filter: + bool: + filter: + - term: + tags.keyword: urn:li:tag:primary + - term: + tags.keyword: urn:li:tag:gold + weight: 3.0 + + + score_mode: multiply + boost_mode: multiply +``` + ##### Example 2: Preferred Data Platform Boost the `urn:li:dataPlatform:hive` platform. From 17c9fcf4fb74e333397c231e37b60f5315f1377f Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Fri, 15 Nov 2024 16:10:26 -0800 Subject: [PATCH 010/174] Update CSVInfo.tsx (#11871) --- datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx b/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx index 87d632bb228b5f..f533890e17fa5c 100644 --- a/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/CSVInfo.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { Alert } from 'antd'; -const CSV_FORMAT_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/csv'; +const CSV_FORMAT_LINK = 'https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher'; export const CSVInfo = () => { const link = ( From 3e128f428648f4ddccda604f0f0219fc684fba1b Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 16 Nov 2024 13:45:39 -0800 Subject: [PATCH 011/174] chore(structured-properties): add cli validation for entity types (#11863) --- .../structured_properties.yaml | 4 +- .../structuredproperties.py | 232 +++++++++++------- .../bad_entity_type.yaml | 6 + .../test_structured_properties.py | 18 ++ .../test_structured_properties.yaml | 10 +- 5 files changed, 169 insertions(+), 101 deletions(-) create mode 100644 smoke-test/tests/structured_properties/bad_entity_type.yaml diff --git a/metadata-ingestion/examples/structured_properties/structured_properties.yaml b/metadata-ingestion/examples/structured_properties/structured_properties.yaml index 5c7ce47ba3b8a5..0230fbcc61f11b 100644 --- a/metadata-ingestion/examples/structured_properties/structured_properties.yaml +++ b/metadata-ingestion/examples/structured_properties/structured_properties.yaml @@ -7,7 +7,7 @@ entity_types: - dataset # or urn:li:entityType:datahub.dataset - dataFlow - description: "Retention Time is used to figure out how long to retain records in a dataset" + description: 'Retention Time is used to figure out how long to retain records in a dataset' allowed_values: - value: 30 description: 30 days, usually reserved for datasets that are ephemeral and contain pii @@ -18,7 +18,7 @@ - id: io.acryl.dataManagement.replicationSLA type: number display_name: Replication SLA - description: "SLA for how long data can be delayed before replicating to the destination cluster" + description: 'SLA for how long data can be delayed before replicating to the destination cluster' entity_types: - dataset - id: io.acryl.dataManagement.deprecationDate diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py index 060d6b00b1a12b..b48c655015d825 100644 --- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py +++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py @@ -1,7 +1,8 @@ import logging +from contextlib import contextmanager from enum import Enum from pathlib import Path -from typing import List, Optional +from typing import Generator, List, Optional import yaml from pydantic import validator @@ -20,6 +21,28 @@ logger = logging.getLogger(__name__) +class StructuredPropertiesConfig: + """Configuration class to hold the graph client""" + + _graph: Optional[DataHubGraph] = None + + @classmethod + @contextmanager + def use_graph(cls, graph: DataHubGraph) -> Generator[None, None, None]: + """Context manager to temporarily set a custom graph""" + previous_graph = cls._graph + cls._graph = graph + try: + yield + finally: + cls._graph = previous_graph + + @classmethod + def get_graph(cls) -> DataHubGraph: + """Get the current graph, falling back to default if none set""" + return cls._graph if cls._graph is not None else get_default_graph() + + class AllowedTypes(Enum): STRING = "string" RICH_TEXT = "rich_text" @@ -41,25 +64,28 @@ class AllowedValue(ConfigModel): description: Optional[str] = None +VALID_ENTITY_TYPES_PREFIX_STRING = ", ".join( + [ + f"urn:li:entityType:datahub.{x}" + for x in ["dataset", "dashboard", "dataFlow", "schemaField"] + ] +) +VALID_ENTITY_TYPES_STRING = f"Valid entity type urns are {VALID_ENTITY_TYPES_PREFIX_STRING}, etc... Ensure that the entity type is valid." + + class TypeQualifierAllowedTypes(ConfigModel): allowed_types: List[str] - @validator("allowed_types") + @validator("allowed_types", each_item=True) def validate_allowed_types(cls, v): - validated_entity_type_urns = [] if v: - with get_default_graph() as graph: - for et in v: - validated_urn = Urn.make_entity_type_urn(et) - if graph.exists(validated_urn): - validated_entity_type_urns.append(validated_urn) - else: - logger.warn( - f"Input {et} is not a valid entity type urn. Skipping." - ) - v = validated_entity_type_urns - if not v: - logger.warn("No allowed_types given within type_qualifier.") + graph = StructuredPropertiesConfig.get_graph() + validated_urn = Urn.make_entity_type_urn(v) + if not graph.exists(validated_urn): + raise ValueError( + f"Input {v} is not a valid entity type urn. {VALID_ENTITY_TYPES_STRING}" + ) + v = str(validated_urn) return v @@ -77,6 +103,18 @@ class StructuredProperties(ConfigModel): type_qualifier: Optional[TypeQualifierAllowedTypes] = None immutable: Optional[bool] = False + @validator("entity_types", each_item=True) + def validate_entity_types(cls, v): + if v: + graph = StructuredPropertiesConfig.get_graph() + validated_urn = Urn.make_entity_type_urn(v) + if not graph.exists(validated_urn): + raise ValueError( + f"Input {v} is not a valid entity type urn. {VALID_ENTITY_TYPES_STRING}" + ) + v = str(validated_urn) + return v + @property def fqn(self) -> str: assert self.urn is not None @@ -97,93 +135,99 @@ def urn_must_be_present(cls, v, values): @staticmethod def create(file: str, graph: Optional[DataHubGraph] = None) -> None: emitter: DataHubGraph = graph if graph else get_default_graph() - - with open(file) as fp: - structuredproperties: List[dict] = yaml.safe_load(fp) - for structuredproperty_raw in structuredproperties: - structuredproperty = StructuredProperties.parse_obj( - structuredproperty_raw - ) - if not structuredproperty.type.islower(): - structuredproperty.type = structuredproperty.type.lower() - logger.warn( - f"Structured property type should be lowercase. Updated to {structuredproperty.type}" + with StructuredPropertiesConfig.use_graph(emitter): + print("Using graph") + with open(file) as fp: + structuredproperties: List[dict] = yaml.safe_load(fp) + for structuredproperty_raw in structuredproperties: + structuredproperty = StructuredProperties.parse_obj( + structuredproperty_raw ) - if not AllowedTypes.check_allowed_type(structuredproperty.type): - raise ValueError( - f"Type {structuredproperty.type} is not allowed. Allowed types are {AllowedTypes.values()}" - ) - mcp = MetadataChangeProposalWrapper( - entityUrn=structuredproperty.urn, - aspect=StructuredPropertyDefinitionClass( - qualifiedName=structuredproperty.fqn, - valueType=Urn.make_data_type_urn(structuredproperty.type), - displayName=structuredproperty.display_name, - description=structuredproperty.description, - entityTypes=[ - Urn.make_entity_type_urn(entity_type) - for entity_type in structuredproperty.entity_types or [] - ], - cardinality=structuredproperty.cardinality, - immutable=structuredproperty.immutable, - allowedValues=( - [ - PropertyValueClass( - value=v.value, description=v.description - ) - for v in structuredproperty.allowed_values - ] - if structuredproperty.allowed_values - else None - ), - typeQualifier=( - { - "allowedTypes": structuredproperty.type_qualifier.allowed_types - } - if structuredproperty.type_qualifier - else None + if not structuredproperty.type.islower(): + structuredproperty.type = structuredproperty.type.lower() + logger.warn( + f"Structured property type should be lowercase. Updated to {structuredproperty.type}" + ) + if not AllowedTypes.check_allowed_type(structuredproperty.type): + raise ValueError( + f"Type {structuredproperty.type} is not allowed. Allowed types are {AllowedTypes.values()}" + ) + mcp = MetadataChangeProposalWrapper( + entityUrn=structuredproperty.urn, + aspect=StructuredPropertyDefinitionClass( + qualifiedName=structuredproperty.fqn, + valueType=Urn.make_data_type_urn(structuredproperty.type), + displayName=structuredproperty.display_name, + description=structuredproperty.description, + entityTypes=[ + Urn.make_entity_type_urn(entity_type) + for entity_type in structuredproperty.entity_types or [] + ], + cardinality=structuredproperty.cardinality, + immutable=structuredproperty.immutable, + allowedValues=( + [ + PropertyValueClass( + value=v.value, description=v.description + ) + for v in structuredproperty.allowed_values + ] + if structuredproperty.allowed_values + else None + ), + typeQualifier=( + { + "allowedTypes": structuredproperty.type_qualifier.allowed_types + } + if structuredproperty.type_qualifier + else None + ), ), - ), - ) - emitter.emit_mcp(mcp) + ) + emitter.emit_mcp(mcp) - logger.info(f"Created structured property {structuredproperty.urn}") + logger.info(f"Created structured property {structuredproperty.urn}") @classmethod def from_datahub(cls, graph: DataHubGraph, urn: str) -> "StructuredProperties": - structured_property: Optional[ - StructuredPropertyDefinitionClass - ] = graph.get_aspect(urn, StructuredPropertyDefinitionClass) - if structured_property is None: - raise Exception( - "StructuredPropertyDefinition aspect is None. Unable to create structured property." + with StructuredPropertiesConfig.use_graph(graph): + structured_property: Optional[ + StructuredPropertyDefinitionClass + ] = graph.get_aspect(urn, StructuredPropertyDefinitionClass) + if structured_property is None: + raise Exception( + "StructuredPropertyDefinition aspect is None. Unable to create structured property." + ) + return StructuredProperties( + urn=urn, + qualified_name=structured_property.qualifiedName, + display_name=structured_property.displayName, + type=structured_property.valueType, + description=structured_property.description, + entity_types=structured_property.entityTypes, + cardinality=structured_property.cardinality, + allowed_values=( + [ + AllowedValue( + value=av.value, + description=av.description, + ) + for av in structured_property.allowedValues or [] + ] + if structured_property.allowedValues is not None + else None + ), + type_qualifier=( + { + "allowed_types": structured_property.typeQualifier.get( + "allowedTypes" + ) + } + if structured_property.typeQualifier + else None + ), ) - return StructuredProperties( - urn=urn, - qualified_name=structured_property.qualifiedName, - display_name=structured_property.displayName, - type=structured_property.valueType, - description=structured_property.description, - entity_types=structured_property.entityTypes, - cardinality=structured_property.cardinality, - allowed_values=( - [ - AllowedValue( - value=av.value, - description=av.description, - ) - for av in structured_property.allowedValues or [] - ] - if structured_property.allowedValues is not None - else None - ), - type_qualifier=( - {"allowed_types": structured_property.typeQualifier.get("allowedTypes")} - if structured_property.typeQualifier - else None - ), - ) def to_yaml( self, diff --git a/smoke-test/tests/structured_properties/bad_entity_type.yaml b/smoke-test/tests/structured_properties/bad_entity_type.yaml new file mode 100644 index 00000000000000..7e919a5a383907 --- /dev/null +++ b/smoke-test/tests/structured_properties/bad_entity_type.yaml @@ -0,0 +1,6 @@ +- id: clusterTypeBad + type: STRING + display_name: Cluster's type + description: 'Test Cluster Type Property' + entity_types: + - urn:li:entityType:dataset # should fail because this is not a valid entity type diff --git a/smoke-test/tests/structured_properties/test_structured_properties.py b/smoke-test/tests/structured_properties/test_structured_properties.py index f0509aad77625f..533a03a55735a1 100644 --- a/smoke-test/tests/structured_properties/test_structured_properties.py +++ b/smoke-test/tests/structured_properties/test_structured_properties.py @@ -371,6 +371,24 @@ def test_structured_property_schema_field(ingest_cleanup_data, graph_client): raise e +def test_structured_properties_yaml_load_with_bad_entity_type( + ingest_cleanup_data, graph_client +): + try: + StructuredProperties.create( + "tests/structured_properties/bad_entity_type.yaml", + graph=graph_client, + ) + raise AssertionError( + "Should not be able to create structured properties with bad entity type" + ) + except Exception as e: + if "urn:li:entityType:dataset is not a valid entity type urn" in str(e): + pass + else: + raise e + + def test_dataset_yaml_loader(ingest_cleanup_data, graph_client): StructuredProperties.create( "tests/structured_properties/test_structured_properties.yaml", diff --git a/smoke-test/tests/structured_properties/test_structured_properties.yaml b/smoke-test/tests/structured_properties/test_structured_properties.yaml index 569a3d185165d6..eee8d53aeccf5a 100644 --- a/smoke-test/tests/structured_properties/test_structured_properties.yaml +++ b/smoke-test/tests/structured_properties/test_structured_properties.yaml @@ -1,13 +1,13 @@ - id: clusterType type: STRING display_name: Cluster's type - description: "Test Cluster Type Property" + description: 'Test Cluster Type Property' entity_types: - dataset - id: clusterName type: STRING display_name: Cluster's name - description: "Test Cluster Name Property" + description: 'Test Cluster Name Property' entity_types: - dataset - id: projectNames @@ -15,9 +15,9 @@ cardinality: MULTIPLE display_name: Project Name entity_types: - - dataset # or urn:li:logicalEntity:metamodel.datahub.dataset - - dataflow - description: "Test property for project name" + - dataset # or urn:li:entityType:datahub.dataset + - dataFlow + description: 'Test property for project name' allowed_values: - value: Tracking description: test value 1 for project From 37fa07694fbfa756d3be72fcf478dfa5e1579823 Mon Sep 17 00:00:00 2001 From: Hendrik Richert Date: Mon, 18 Nov 2024 12:25:40 +0100 Subject: [PATCH 012/174] feat(py-sdk): add cli version to ingestion headers (#11847) Co-authored-by: Hendrik Richert Co-authored-by: Shirshanka Das --- metadata-ingestion/src/datahub/emitter/rest_emitter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-ingestion/src/datahub/emitter/rest_emitter.py b/metadata-ingestion/src/datahub/emitter/rest_emitter.py index 948060c3c4f44c..ef2082b95330b4 100644 --- a/metadata-ingestion/src/datahub/emitter/rest_emitter.py +++ b/metadata-ingestion/src/datahub/emitter/rest_emitter.py @@ -10,6 +10,7 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import HTTPError, RequestException +from datahub import nice_version_name from datahub.cli import config_utils from datahub.cli.cli_utils import ensure_has_system_metadata, fixup_gms_url from datahub.configuration.common import ConfigurationError, OperationalError @@ -91,6 +92,7 @@ def __init__( self._session.headers.update( { "X-RestLi-Protocol-Version": "2.0.0", + "X-DataHub-Py-Cli-Version": nice_version_name(), "Content-Type": "application/json", } ) From d660db11deab1659214931a26ecc9ffd6aa44306 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Mon, 18 Nov 2024 13:36:39 +0100 Subject: [PATCH 013/174] fix(ingest/airflow): Remove seems like not intended force debug mode which caused plugin fail on EMR (#11877) --- metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py b/metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py index fc3f877ede6299..55f30b576b44ef 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py +++ b/metadata-ingestion/src/datahub/sql_parsing/_sqlglot_patch.py @@ -19,7 +19,7 @@ # For a diff-formatted view, see: # https://github.com/tobymao/sqlglot/compare/main...hsheth2:sqlglot:main.diff -_DEBUG_PATCHER = is_pytest_running() or True +_DEBUG_PATCHER = is_pytest_running() logger = logging.getLogger(__name__) _apply_diff_subprocess = patchy.api._apply_patch From 435792ca2e6bdfd18920c17a54a94a2ddf817791 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Mon, 18 Nov 2024 17:00:22 +0100 Subject: [PATCH 014/174] fix(ingest/airflow): Add log to dag emit (#11880) --- .../src/datahub_airflow_plugin/datahub_listener.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py index c1d5b306f187dc..e00cf51ea456cc 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py @@ -570,6 +570,9 @@ def on_task_instance_failed( def on_dag_start(self, dag_run: "DagRun") -> None: dag = dag_run.dag if not dag: + logger.warning( + f"DataHub listener could not find DAG for {dag_run.dag_id} - {dag_run.run_id}. Dag won't be captured" + ) return dataflow = AirflowGenerator.generate_dataflow( @@ -577,6 +580,7 @@ def on_dag_start(self, dag_run: "DagRun") -> None: dag=dag, ) dataflow.emit(self.emitter, callback=self._make_emit_callback()) + logger.debug(f"Emitted DataHub DataFlow: {dataflow}") event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( entityUrn=str(dataflow.urn), aspect=StatusClass(removed=False) From 2527f54972e58f5053b425bbef6b906e3c7c924c Mon Sep 17 00:00:00 2001 From: skrydal Date: Mon, 18 Nov 2024 19:41:45 +0100 Subject: [PATCH 015/174] feat(ingest/iceberg): Iceberg performance improvement (multi-threading) (#11182) --- .../ingestion/source/iceberg/iceberg.py | 178 +++++-- .../source/iceberg/iceberg_common.py | 60 +++ .../source/iceberg/iceberg_profiler.py | 169 +++--- .../utilities/threaded_iterator_executor.py | 1 - .../iceberg_multiprocessing_to_file.yml | 22 + .../tests/integration/iceberg/test_iceberg.py | 64 ++- metadata-ingestion/tests/unit/test_iceberg.py | 479 +++++++++++++++++- 7 files changed, 821 insertions(+), 152 deletions(-) create mode 100644 metadata-ingestion/tests/integration/iceberg/iceberg_multiprocessing_to_file.yml diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py index d8c6c03ce81e67..258a4b9ad6daf6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py @@ -1,10 +1,15 @@ import json import logging +import threading import uuid from typing import Any, Dict, Iterable, List, Optional from pyiceberg.catalog import Catalog -from pyiceberg.exceptions import NoSuchIcebergTableError +from pyiceberg.exceptions import ( + NoSuchIcebergTableError, + NoSuchNamespaceError, + NoSuchPropertyException, +) from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType, visit from pyiceberg.table import Table from pyiceberg.typedef import Identifier @@ -75,6 +80,8 @@ OwnershipClass, OwnershipTypeClass, ) +from datahub.utilities.perf_timer import PerfTimer +from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor LOGGER = logging.getLogger(__name__) logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel( @@ -130,74 +137,149 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: ] def _get_datasets(self, catalog: Catalog) -> Iterable[Identifier]: - for namespace in catalog.list_namespaces(): - yield from catalog.list_tables(namespace) + namespaces = catalog.list_namespaces() + LOGGER.debug( + f"Retrieved {len(namespaces)} namespaces, first 10: {namespaces[:10]}" + ) + self.report.report_no_listed_namespaces(len(namespaces)) + tables_count = 0 + for namespace in namespaces: + try: + tables = catalog.list_tables(namespace) + tables_count += len(tables) + LOGGER.debug( + f"Retrieved {len(tables)} tables for namespace: {namespace}, in total retrieved {tables_count}, first 10: {tables[:10]}" + ) + self.report.report_listed_tables_for_namespace( + ".".join(namespace), len(tables) + ) + yield from tables + except NoSuchNamespaceError: + self.report.report_warning( + "no-such-namespace", + f"Couldn't list tables for namespace {namespace} due to NoSuchNamespaceError exception", + ) + LOGGER.warning( + f"NoSuchNamespaceError exception while trying to get list of tables from namespace {namespace}, skipping it", + ) + except Exception as e: + self.report.report_failure( + "listing-tables-exception", + f"Couldn't list tables for namespace {namespace} due to {e}", + ) + LOGGER.exception( + f"Unexpected exception while trying to get list of tables for namespace {namespace}, skipping it" + ) def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: - try: - catalog = self.config.get_catalog() - except Exception as e: - LOGGER.error("Failed to get catalog", exc_info=True) - self.report.report_failure("get-catalog", f"Failed to get catalog: {e}") - return + thread_local = threading.local() - for dataset_path in self._get_datasets(catalog): + def _process_dataset(dataset_path: Identifier) -> Iterable[MetadataWorkUnit]: + LOGGER.debug(f"Processing dataset for path {dataset_path}") dataset_name = ".".join(dataset_path) if not self.config.table_pattern.allowed(dataset_name): # Dataset name is rejected by pattern, report as dropped. self.report.report_dropped(dataset_name) - continue - + return try: - # Try to load an Iceberg table. Might not contain one, this will be caught by NoSuchIcebergTableError. - table = catalog.load_table(dataset_path) + if not hasattr(thread_local, "local_catalog"): + LOGGER.debug( + f"Didn't find local_catalog in thread_local ({thread_local}), initializing new catalog" + ) + thread_local.local_catalog = self.config.get_catalog() + + with PerfTimer() as timer: + table = thread_local.local_catalog.load_table(dataset_path) + time_taken = timer.elapsed_seconds() + self.report.report_table_load_time(time_taken) + LOGGER.debug( + f"Loaded table: {table.identifier}, time taken: {time_taken}" + ) yield from self._create_iceberg_workunit(dataset_name, table) + except NoSuchPropertyException as e: + self.report.report_warning( + "table-property-missing", + f"Failed to create workunit for {dataset_name}. {e}", + ) + LOGGER.warning( + f"NoSuchPropertyException while processing table {dataset_path}, skipping it.", + ) except NoSuchIcebergTableError as e: + self.report.report_warning( + "no-iceberg-table", + f"Failed to create workunit for {dataset_name}. {e}", + ) + LOGGER.warning( + f"NoSuchIcebergTableError while processing table {dataset_path}, skipping it.", + ) + except Exception as e: self.report.report_failure("general", f"Failed to create workunit: {e}") LOGGER.exception( f"Exception while processing table {dataset_path}, skipping it.", ) + try: + catalog = self.config.get_catalog() + except Exception as e: + self.report.report_failure("get-catalog", f"Failed to get catalog: {e}") + return + + for wu in ThreadedIteratorExecutor.process( + worker_func=_process_dataset, + args_list=[(dataset_path,) for dataset_path in self._get_datasets(catalog)], + max_workers=self.config.processing_threads, + ): + yield wu + def _create_iceberg_workunit( self, dataset_name: str, table: Table ) -> Iterable[MetadataWorkUnit]: - self.report.report_table_scanned(dataset_name) - dataset_urn: str = make_dataset_urn_with_platform_instance( - self.platform, - dataset_name, - self.config.platform_instance, - self.config.env, - ) - dataset_snapshot = DatasetSnapshot( - urn=dataset_urn, - aspects=[Status(removed=False)], - ) - - # Dataset properties aspect. - custom_properties = table.metadata.properties.copy() - custom_properties["location"] = table.metadata.location - custom_properties["format-version"] = str(table.metadata.format_version) - custom_properties["partition-spec"] = str(self._get_partition_aspect(table)) - if table.current_snapshot(): - custom_properties["snapshot-id"] = str(table.current_snapshot().snapshot_id) - custom_properties["manifest-list"] = table.current_snapshot().manifest_list - dataset_properties = DatasetPropertiesClass( - name=table.name()[-1], - tags=[], - description=table.metadata.properties.get("comment", None), - customProperties=custom_properties, - ) - dataset_snapshot.aspects.append(dataset_properties) + with PerfTimer() as timer: + self.report.report_table_scanned(dataset_name) + LOGGER.debug(f"Processing table {dataset_name}") + dataset_urn: str = make_dataset_urn_with_platform_instance( + self.platform, + dataset_name, + self.config.platform_instance, + self.config.env, + ) + dataset_snapshot = DatasetSnapshot( + urn=dataset_urn, + aspects=[Status(removed=False)], + ) - # Dataset ownership aspect. - dataset_ownership = self._get_ownership_aspect(table) - if dataset_ownership: - dataset_snapshot.aspects.append(dataset_ownership) + # Dataset properties aspect. + custom_properties = table.metadata.properties.copy() + custom_properties["location"] = table.metadata.location + custom_properties["format-version"] = str(table.metadata.format_version) + custom_properties["partition-spec"] = str(self._get_partition_aspect(table)) + if table.current_snapshot(): + custom_properties["snapshot-id"] = str( + table.current_snapshot().snapshot_id + ) + custom_properties[ + "manifest-list" + ] = table.current_snapshot().manifest_list + dataset_properties = DatasetPropertiesClass( + name=table.name()[-1], + tags=[], + description=table.metadata.properties.get("comment", None), + customProperties=custom_properties, + ) + dataset_snapshot.aspects.append(dataset_properties) + # Dataset ownership aspect. + dataset_ownership = self._get_ownership_aspect(table) + if dataset_ownership: + LOGGER.debug( + f"Adding ownership: {dataset_ownership} to the dataset {dataset_name}" + ) + dataset_snapshot.aspects.append(dataset_ownership) - schema_metadata = self._create_schema_metadata(dataset_name, table) - dataset_snapshot.aspects.append(schema_metadata) + schema_metadata = self._create_schema_metadata(dataset_name, table) + dataset_snapshot.aspects.append(schema_metadata) - mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + self.report.report_table_processing_time(timer.elapsed_seconds()) yield MetadataWorkUnit(id=dataset_name, mce=mce) dpi_aspect = self._get_dataplatform_instance_aspect(dataset_urn=dataset_urn) diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py index b74c096d0798e8..98ad9e552d35c9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional +from humanfriendly import format_timespan from pydantic import Field, validator from pyiceberg.catalog import Catalog, load_catalog @@ -18,6 +19,7 @@ OperationConfig, is_profiling_enabled, ) +from datahub.utilities.stats_collections import TopKDict, int_top_k_dict logger = logging.getLogger(__name__) @@ -75,6 +77,9 @@ class IcebergSourceConfig(StatefulIngestionConfigBase, DatasetSourceConfigMixin) description="Iceberg table property to look for a `CorpGroup` owner. Can only hold a single group value. If property has no value, no owner information will be emitted.", ) profiling: IcebergProfilingConfig = IcebergProfilingConfig() + processing_threads: int = Field( + default=1, description="How many threads will be processing tables" + ) @validator("catalog", pre=True, always=True) def handle_deprecated_catalog_format(cls, value): @@ -131,17 +136,72 @@ def get_catalog(self) -> Catalog: # Retrieve the dict associated with the one catalog entry catalog_name, catalog_config = next(iter(self.catalog.items())) + logger.debug( + "Initializing the catalog %s with config: %s", catalog_name, catalog_config + ) return load_catalog(name=catalog_name, **catalog_config) +class TimingClass: + times: List[int] + + def __init__(self): + self.times = [] + + def add_timing(self, t): + self.times.append(t) + + def __str__(self): + if len(self.times) == 0: + return "no timings reported" + self.times.sort() + total = sum(self.times) + avg = total / len(self.times) + return str( + { + "average_time": format_timespan(avg, detailed=True, max_units=3), + "min_time": format_timespan(self.times[0], detailed=True, max_units=3), + "max_time": format_timespan(self.times[-1], detailed=True, max_units=3), + # total_time does not provide correct information in case we run in more than 1 thread + "total_time": format_timespan(total, detailed=True, max_units=3), + } + ) + + @dataclass class IcebergSourceReport(StaleEntityRemovalSourceReport): tables_scanned: int = 0 entities_profiled: int = 0 filtered: List[str] = field(default_factory=list) + load_table_timings: TimingClass = field(default_factory=TimingClass) + processing_table_timings: TimingClass = field(default_factory=TimingClass) + profiling_table_timings: TimingClass = field(default_factory=TimingClass) + listed_namespaces: int = 0 + total_listed_tables: int = 0 + tables_listed_per_namespace: TopKDict[str, int] = field( + default_factory=int_top_k_dict + ) + + def report_listed_tables_for_namespace( + self, namespace: str, no_tables: int + ) -> None: + self.tables_listed_per_namespace[namespace] = no_tables + self.total_listed_tables += no_tables + + def report_no_listed_namespaces(self, amount: int) -> None: + self.listed_namespaces = amount def report_table_scanned(self, name: str) -> None: self.tables_scanned += 1 def report_dropped(self, ent_name: str) -> None: self.filtered.append(ent_name) + + def report_table_load_time(self, t: float) -> None: + self.load_table_timings.add_timing(t) + + def report_table_processing_time(self, t: float) -> None: + self.processing_table_timings.add_timing(t) + + def report_table_profiling_time(self, t: float) -> None: + self.profiling_table_timings.add_timing(t) diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py index e1d52752d779a0..9cc6dd08544e4e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_profiler.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Callable, Dict, Iterable, Union, cast from pyiceberg.conversions import from_bytes @@ -33,6 +34,9 @@ DatasetFieldProfileClass, DatasetProfileClass, ) +from datahub.utilities.perf_timer import PerfTimer + +LOGGER = logging.getLogger(__name__) class IcebergProfiler: @@ -109,92 +113,101 @@ def profile_table( Yields: Iterator[Iterable[MetadataWorkUnit]]: Workunits related to datasetProfile. """ - current_snapshot = table.current_snapshot() - if not current_snapshot: - # Table has no data, cannot profile, or we can't get current_snapshot. - return - - row_count = ( - int(current_snapshot.summary.additional_properties["total-records"]) - if current_snapshot.summary - else 0 - ) - column_count = len( - [ - field.field_id - for field in table.schema().fields - if field.field_type.is_primitive - ] - ) - dataset_profile = DatasetProfileClass( - timestampMillis=get_sys_time(), - rowCount=row_count, - columnCount=column_count, - ) - dataset_profile.fieldProfiles = [] + with PerfTimer() as timer: + LOGGER.debug(f"Starting profiling of dataset: {dataset_name}") + current_snapshot = table.current_snapshot() + if not current_snapshot: + # Table has no data, cannot profile, or we can't get current_snapshot. + return + + row_count = ( + int(current_snapshot.summary.additional_properties["total-records"]) + if current_snapshot.summary + else 0 + ) + column_count = len( + [ + field.field_id + for field in table.schema().fields + if field.field_type.is_primitive + ] + ) + dataset_profile = DatasetProfileClass( + timestampMillis=get_sys_time(), + rowCount=row_count, + columnCount=column_count, + ) + dataset_profile.fieldProfiles = [] - total_count = 0 - null_counts: Dict[int, int] = {} - min_bounds: Dict[int, Any] = {} - max_bounds: Dict[int, Any] = {} - try: - for manifest in current_snapshot.manifests(table.io): - for manifest_entry in manifest.fetch_manifest_entry(table.io): - data_file = manifest_entry.data_file + total_count = 0 + null_counts: Dict[int, int] = {} + min_bounds: Dict[int, Any] = {} + max_bounds: Dict[int, Any] = {} + try: + for manifest in current_snapshot.manifests(table.io): + for manifest_entry in manifest.fetch_manifest_entry(table.io): + data_file = manifest_entry.data_file + if self.config.include_field_null_count: + null_counts = self._aggregate_counts( + null_counts, data_file.null_value_counts + ) + if self.config.include_field_min_value: + self._aggregate_bounds( + table.schema(), + min, + min_bounds, + data_file.lower_bounds, + ) + if self.config.include_field_max_value: + self._aggregate_bounds( + table.schema(), + max, + max_bounds, + data_file.upper_bounds, + ) + total_count += data_file.record_count + except Exception as e: + # Catch any errors that arise from attempting to read the Iceberg table's manifests + # This will prevent stateful ingestion from being blocked by an error (profiling is not critical) + self.report.report_warning( + "profiling", + f"Error while profiling dataset {dataset_name}: {e}", + ) + if row_count: + # Iterating through fieldPaths introduces unwanted stats for list element fields... + for field_path, field_id in table.schema()._name_to_id.items(): + field = table.schema().find_field(field_id) + column_profile = DatasetFieldProfileClass(fieldPath=field_path) if self.config.include_field_null_count: - null_counts = self._aggregate_counts( - null_counts, data_file.null_value_counts + column_profile.nullCount = cast( + int, null_counts.get(field_id, 0) + ) + column_profile.nullProportion = float( + column_profile.nullCount / row_count ) + if self.config.include_field_min_value: - self._aggregate_bounds( - table.schema(), - min, - min_bounds, - data_file.lower_bounds, + column_profile.min = ( + self._render_value( + dataset_name, field.field_type, min_bounds.get(field_id) + ) + if field_id in min_bounds + else None ) if self.config.include_field_max_value: - self._aggregate_bounds( - table.schema(), - max, - max_bounds, - data_file.upper_bounds, + column_profile.max = ( + self._render_value( + dataset_name, field.field_type, max_bounds.get(field_id) + ) + if field_id in max_bounds + else None ) - total_count += data_file.record_count - except Exception as e: - # Catch any errors that arise from attempting to read the Iceberg table's manifests - # This will prevent stateful ingestion from being blocked by an error (profiling is not critical) - self.report.report_warning( - "profiling", - f"Error while profiling dataset {dataset_name}: {e}", + dataset_profile.fieldProfiles.append(column_profile) + time_taken = timer.elapsed_seconds() + self.report.report_table_profiling_time(time_taken) + LOGGER.debug( + f"Finished profiling of dataset: {dataset_name} in {time_taken}" ) - if row_count: - # Iterating through fieldPaths introduces unwanted stats for list element fields... - for field_path, field_id in table.schema()._name_to_id.items(): - field = table.schema().find_field(field_id) - column_profile = DatasetFieldProfileClass(fieldPath=field_path) - if self.config.include_field_null_count: - column_profile.nullCount = cast(int, null_counts.get(field_id, 0)) - column_profile.nullProportion = float( - column_profile.nullCount / row_count - ) - - if self.config.include_field_min_value: - column_profile.min = ( - self._render_value( - dataset_name, field.field_type, min_bounds.get(field_id) - ) - if field_id in min_bounds - else None - ) - if self.config.include_field_max_value: - column_profile.max = ( - self._render_value( - dataset_name, field.field_type, max_bounds.get(field_id) - ) - if field_id in max_bounds - else None - ) - dataset_profile.fieldProfiles.append(column_profile) yield MetadataChangeProposalWrapper( entityUrn=dataset_urn, diff --git a/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py b/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py index 216fa155035d3e..4d328ad31c6c4a 100644 --- a/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py +++ b/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py @@ -46,7 +46,6 @@ def _worker_wrapper( futures = [f for f in futures if not f.done()] if not futures: break - # Yield the remaining work units. This theoretically should not happen, but adding it just in case. while not out_q.empty(): yield out_q.get_nowait() diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_multiprocessing_to_file.yml b/metadata-ingestion/tests/integration/iceberg/iceberg_multiprocessing_to_file.yml new file mode 100644 index 00000000000000..e5e866fb561c9b --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_multiprocessing_to_file.yml @@ -0,0 +1,22 @@ +run_id: iceberg-test + +source: + type: iceberg + config: + processing_threads: 5 + catalog: + default: + type: rest + uri: http://localhost:8181 + s3.access-key-id: admin + s3.secret-access-key: password + s3.region: us-east-1 + warehouse: s3a://warehouse/wh/ + s3.endpoint: http://localhost:9000 + user_ownership_property: owner + group_ownership_property: owner + +sink: + type: file + config: + filename: "./iceberg_mces.json" diff --git a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py index 5a12afa457f01a..85809e557dd8d3 100644 --- a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py +++ b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py @@ -1,5 +1,5 @@ import subprocess -from typing import Any, Dict, List +from typing import Any, Dict from unittest.mock import patch import pytest @@ -18,6 +18,13 @@ FROZEN_TIME = "2020-04-14 07:00:00" GMS_PORT = 8080 GMS_SERVER = f"http://localhost:{GMS_PORT}" +# These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to +# these. +PATHS_IN_GOLDEN_FILE_TO_IGNORE = [ + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]", + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]", + r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]", +] @pytest.fixture(autouse=True, scope="module") @@ -35,6 +42,36 @@ def spark_submit(file_path: str, args: str = "") -> None: assert ret.returncode == 0 +@freeze_time(FROZEN_TIME) +def test_multiprocessing_iceberg_ingest( + docker_compose_runner, pytestconfig, tmp_path, mock_time +): + test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/" + + with docker_compose_runner( + test_resources_dir / "docker-compose.yml", "iceberg" + ) as docker_services: + wait_for_port(docker_services, "spark-iceberg", 8888, timeout=120) + + # Run the create.py pyspark file to populate the table. + spark_submit("/home/iceberg/setup/create.py", "nyc.taxis") + + # Run the metadata ingestion pipeline. + config_file = ( + test_resources_dir / "iceberg_multiprocessing_to_file.yml" + ).resolve() + run_datahub_cmd( + ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path + ) + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE, + output_path=tmp_path / "iceberg_mces.json", + golden_path=test_resources_dir / "iceberg_ingest_mces_golden.json", + ) + + @freeze_time(FROZEN_TIME) def test_iceberg_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/" @@ -52,16 +89,10 @@ def test_iceberg_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time run_datahub_cmd( ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path ) - # These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to these. - ignore_paths: List[str] = [ - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]", - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]", - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]", - ] # Verify the output. mce_helpers.check_golden_file( pytestconfig, - ignore_paths=ignore_paths, + ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE, output_path=tmp_path / "iceberg_mces.json", golden_path=test_resources_dir / "iceberg_ingest_mces_golden.json", ) @@ -170,16 +201,10 @@ def test_iceberg_stateful_ingest( pipeline=pipeline_run2, expected_providers=1 ) - ignore_paths: List[str] = [ - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]", - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]", - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]", - ] - # Verify the output. mce_helpers.check_golden_file( pytestconfig, - ignore_paths=ignore_paths, + ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE, output_path=deleted_mces_path, golden_path=test_resources_dir / "iceberg_deleted_table_mces_golden.json", ) @@ -202,16 +227,11 @@ def test_iceberg_profiling(docker_compose_runner, pytestconfig, tmp_path, mock_t run_datahub_cmd( ["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path ) - # These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to these. - ignore_paths: List[str] = [ - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['created-at'\]", - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['snapshot-id'\]", - r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['manifest-list'\]", - ] + # Verify the output. mce_helpers.check_golden_file( pytestconfig, - ignore_paths=ignore_paths, + ignore_paths=PATHS_IN_GOLDEN_FILE_TO_IGNORE, output_path=tmp_path / "iceberg_mces.json", golden_path=test_resources_dir / "iceberg_profile_mces_golden.json", ) diff --git a/metadata-ingestion/tests/unit/test_iceberg.py b/metadata-ingestion/tests/unit/test_iceberg.py index c8c6c6ac8a85d3..b8a136586a2bf5 100644 --- a/metadata-ingestion/tests/unit/test_iceberg.py +++ b/metadata-ingestion/tests/unit/test_iceberg.py @@ -1,10 +1,21 @@ import uuid from decimal import Decimal -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple +from unittest import TestCase +from unittest.mock import patch import pytest from pydantic import ValidationError +from pyiceberg.exceptions import ( + NoSuchIcebergTableError, + NoSuchNamespaceError, + NoSuchPropertyException, +) +from pyiceberg.io.pyarrow import PyArrowFileIO +from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema +from pyiceberg.table import Table +from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.types import ( BinaryType, BooleanType, @@ -29,16 +40,19 @@ ) from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.iceberg.iceberg import ( IcebergProfiler, IcebergSource, IcebergSourceConfig, ) +from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.com.linkedin.pegasus2avro.schema import ArrayType, SchemaField from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, BytesTypeClass, + DatasetSnapshotClass, DateTypeClass, FixedTypeClass, NumberTypeClass, @@ -48,11 +62,13 @@ ) -def with_iceberg_source() -> IcebergSource: +def with_iceberg_source(processing_threads: int = 1) -> IcebergSource: catalog = {"test": {"type": "rest"}} return IcebergSource( ctx=PipelineContext(run_id="iceberg-source-test"), - config=IcebergSourceConfig(catalog=catalog), + config=IcebergSourceConfig( + catalog=catalog, processing_threads=processing_threads + ), ) @@ -515,3 +531,460 @@ def test_avro_decimal_bytes_nullable() -> None: print( f"After avro parsing, _nullable attribute is preserved: {boolean_avro_schema}" ) + + +class MockCatalog: + def __init__(self, tables: Dict[str, Dict[str, Callable[[], Table]]]): + """ + + :param tables: Dictionary containing namespaces as keys and dictionaries containing names of tables (keys) and + their metadata as values + """ + self.tables = tables + + def list_namespaces(self) -> Iterable[str]: + return [*self.tables.keys()] + + def list_tables(self, namespace: str) -> Iterable[Tuple[str, str]]: + return [(namespace, table) for table in self.tables[namespace].keys()] + + def load_table(self, dataset_path: Tuple[str, str]) -> Table: + return self.tables[dataset_path[0]][dataset_path[1]]() + + +class MockCatalogExceptionListingTables(MockCatalog): + def list_tables(self, namespace: str) -> Iterable[Tuple[str, str]]: + if namespace == "no_such_namespace": + raise NoSuchNamespaceError() + if namespace == "generic_exception": + raise Exception() + return super().list_tables(namespace) + + +class MockCatalogExceptionListingNamespaces(MockCatalog): + def list_namespaces(self) -> Iterable[str]: + raise Exception() + + +def test_exception_while_listing_namespaces() -> None: + source = with_iceberg_source(processing_threads=2) + mock_catalog = MockCatalogExceptionListingNamespaces({}) + with patch( + "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog" + ) as get_catalog, pytest.raises(Exception): + get_catalog.return_value = mock_catalog + [*source.get_workunits_internal()] + + +def test_known_exception_while_listing_tables() -> None: + source = with_iceberg_source(processing_threads=2) + mock_catalog = MockCatalogExceptionListingTables( + { + "namespaceA": { + "table1": lambda: Table( + identifier=("namespaceA", "table1"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table1", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table1", + io=PyArrowFileIO(), + catalog=None, + ) + }, + "no_such_namespace": {}, + "namespaceB": { + "table2": lambda: Table( + identifier=("namespaceB", "table2"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceB/table2", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceB/table2", + io=PyArrowFileIO(), + catalog=None, + ), + "table3": lambda: Table( + identifier=("namespaceB", "table3"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceB/table3", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceB/table3", + io=PyArrowFileIO(), + catalog=None, + ), + }, + "namespaceC": { + "table4": lambda: Table( + identifier=("namespaceC", "table4"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceC/table4", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceC/table4", + io=PyArrowFileIO(), + catalog=None, + ) + }, + "namespaceD": { + "table5": lambda: Table( + identifier=("namespaceD", "table5"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table5", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table5", + io=PyArrowFileIO(), + catalog=None, + ) + }, + } + ) + with patch( + "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog" + ) as get_catalog: + get_catalog.return_value = mock_catalog + wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()] + assert len(wu) == 5 # ingested 5 tables, despite exception + urns = [] + for unit in wu: + assert isinstance(unit.metadata, MetadataChangeEvent) + assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass) + urns.append(unit.metadata.proposedSnapshot.urn) + TestCase().assertCountEqual( + urns, + [ + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)", + ], + ) + assert source.report.warnings.total_elements == 1 + assert source.report.failures.total_elements == 0 + assert source.report.tables_scanned == 5 + + +def test_unknown_exception_while_listing_tables() -> None: + source = with_iceberg_source(processing_threads=2) + mock_catalog = MockCatalogExceptionListingTables( + { + "namespaceA": { + "table1": lambda: Table( + identifier=("namespaceA", "table1"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table1", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table1", + io=PyArrowFileIO(), + catalog=None, + ) + }, + "generic_exception": {}, + "namespaceB": { + "table2": lambda: Table( + identifier=("namespaceB", "table2"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceB/table2", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceB/table2", + io=PyArrowFileIO(), + catalog=None, + ), + "table3": lambda: Table( + identifier=("namespaceB", "table3"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceB/table3", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceB/table3", + io=PyArrowFileIO(), + catalog=None, + ), + }, + "namespaceC": { + "table4": lambda: Table( + identifier=("namespaceC", "table4"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceC/table4", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceC/table4", + io=PyArrowFileIO(), + catalog=None, + ) + }, + "namespaceD": { + "table5": lambda: Table( + identifier=("namespaceD", "table5"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table5", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table5", + io=PyArrowFileIO(), + catalog=None, + ) + }, + } + ) + with patch( + "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog" + ) as get_catalog: + get_catalog.return_value = mock_catalog + wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()] + assert len(wu) == 5 # ingested 5 tables, despite exception + urns = [] + for unit in wu: + assert isinstance(unit.metadata, MetadataChangeEvent) + assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass) + urns.append(unit.metadata.proposedSnapshot.urn) + TestCase().assertCountEqual( + urns, + [ + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table2,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceB.table3,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceC.table4,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceD.table5,PROD)", + ], + ) + assert source.report.warnings.total_elements == 0 + assert source.report.failures.total_elements == 1 + assert source.report.tables_scanned == 5 + + +def test_proper_run_with_multiple_namespaces() -> None: + source = with_iceberg_source(processing_threads=3) + mock_catalog = MockCatalog( + { + "namespaceA": { + "table1": lambda: Table( + identifier=("namespaceA", "table1"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table1", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table1", + io=PyArrowFileIO(), + catalog=None, + ) + }, + "namespaceB": {}, + } + ) + with patch( + "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog" + ) as get_catalog: + get_catalog.return_value = mock_catalog + wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()] + assert len(wu) == 1 # only one table processed as an MCE + assert isinstance(wu[0].metadata, MetadataChangeEvent) + assert isinstance(wu[0].metadata.proposedSnapshot, DatasetSnapshotClass) + snapshot: DatasetSnapshotClass = wu[0].metadata.proposedSnapshot + assert ( + snapshot.urn + == "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)" + ) + + +def test_handle_expected_exceptions() -> None: + source = with_iceberg_source(processing_threads=3) + + def _raise_no_such_property_exception(): + raise NoSuchPropertyException() + + def _raise_no_such_table_exception(): + raise NoSuchIcebergTableError() + + mock_catalog = MockCatalog( + { + "namespaceA": { + "table1": lambda: Table( + identifier=("namespaceA", "table1"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table1", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table1", + io=PyArrowFileIO(), + catalog=None, + ), + "table2": lambda: Table( + identifier=("namespaceA", "table2"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table2", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table2", + io=PyArrowFileIO(), + catalog=None, + ), + "table3": lambda: Table( + identifier=("namespaceA", "table3"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table3", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table3", + io=PyArrowFileIO(), + catalog=None, + ), + "table4": lambda: Table( + identifier=("namespaceA", "table4"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table4", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table4", + io=PyArrowFileIO(), + catalog=None, + ), + "table5": _raise_no_such_property_exception, + "table6": _raise_no_such_table_exception, + } + } + ) + with patch( + "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog" + ) as get_catalog: + get_catalog.return_value = mock_catalog + wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()] + assert len(wu) == 4 + urns = [] + for unit in wu: + assert isinstance(unit.metadata, MetadataChangeEvent) + assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass) + urns.append(unit.metadata.proposedSnapshot.urn) + TestCase().assertCountEqual( + urns, + [ + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)", + ], + ) + assert source.report.warnings.total_elements == 2 + assert source.report.failures.total_elements == 0 + assert source.report.tables_scanned == 4 + + +def test_handle_unexpected_exceptions() -> None: + source = with_iceberg_source(processing_threads=3) + + def _raise_exception(): + raise Exception() + + mock_catalog = MockCatalog( + { + "namespaceA": { + "table1": lambda: Table( + identifier=("namespaceA", "table1"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table1", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table1", + io=PyArrowFileIO(), + catalog=None, + ), + "table2": lambda: Table( + identifier=("namespaceA", "table2"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table2", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table2", + io=PyArrowFileIO(), + catalog=None, + ), + "table3": lambda: Table( + identifier=("namespaceA", "table3"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table3", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table3", + io=PyArrowFileIO(), + catalog=None, + ), + "table4": lambda: Table( + identifier=("namespaceA", "table4"), + metadata=TableMetadataV2( + partition_specs=[PartitionSpec(spec_id=0)], + location="s3://abcdefg/namespaceA/table4", + last_column_id=0, + schemas=[Schema(schema_id=0)], + ), + metadata_location="s3://abcdefg/namespaceA/table4", + io=PyArrowFileIO(), + catalog=None, + ), + "table5": _raise_exception, + } + } + ) + with patch( + "datahub.ingestion.source.iceberg.iceberg.IcebergSourceConfig.get_catalog" + ) as get_catalog: + get_catalog.return_value = mock_catalog + wu: List[MetadataWorkUnit] = [*source.get_workunits_internal()] + assert len(wu) == 4 + urns = [] + for unit in wu: + assert isinstance(unit.metadata, MetadataChangeEvent) + assert isinstance(unit.metadata.proposedSnapshot, DatasetSnapshotClass) + urns.append(unit.metadata.proposedSnapshot.urn) + TestCase().assertCountEqual( + urns, + [ + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table2,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table3,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespaceA.table4,PROD)", + ], + ) + assert source.report.warnings.total_elements == 0 + assert source.report.failures.total_elements == 1 + assert source.report.tables_scanned == 4 From 19702c822580301383a4d0a788c6b526191b391b Mon Sep 17 00:00:00 2001 From: Raudzis Sebastian <32541580+raudzis@users.noreply.github.com> Date: Tue, 19 Nov 2024 00:45:05 +0100 Subject: [PATCH 016/174] fix(ingest/lookml): replace class variable with instance variable for improved encapsulation (#11881) --- .../ingestion/source/looker/lookml_source.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index e4d8dd19fb7917..d258570ec384f7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -283,23 +283,21 @@ class LookMLSource(StatefulIngestionSourceBase): """ platform = "lookml" - source_config: LookMLSourceConfig - reporter: LookMLSourceReport - looker_client: Optional[LookerAPI] = None - - # This is populated during the git clone step. - base_projects_folder: Dict[str, pathlib.Path] = {} - remote_projects_git_info: Dict[str, GitInfo] = {} def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext): super().__init__(config, ctx) - self.source_config = config + self.source_config: LookMLSourceConfig = config self.ctx = ctx self.reporter = LookMLSourceReport() # To keep track of projects (containers) which have already been ingested self.processed_projects: List[str] = [] + # This is populated during the git clone step. + self.base_projects_folder: Dict[str, pathlib.Path] = {} + self.remote_projects_git_info: Dict[str, GitInfo] = {} + + self.looker_client: Optional[LookerAPI] = None if self.source_config.api: self.looker_client = LookerAPI(self.source_config.api) self.reporter._looker_api = self.looker_client From bf16e58d43563636f3a439d0b9ce85ab0cef02ea Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 18 Nov 2024 17:50:04 -0600 Subject: [PATCH 017/174] docs(urn): urn encoding (#11884) --- docs/what/urn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/what/urn.md b/docs/what/urn.md index 122b93038d9dec..e35ca7fbaca4bc 100644 --- a/docs/what/urn.md +++ b/docs/what/urn.md @@ -41,4 +41,4 @@ There are a few restrictions when creating an urn: 2. Parentheses are reserved characters in URN fields: `( , )` 3. Colons are reserved characters in URN fields: `:` -Please do not use these characters when creating or generating urns. +Please do not use these characters when creating or generating urns. One approach is to use URL encoding for the characters. From 94f1f39667a6edf2570fba04a9e1effc60423f9a Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Mon, 18 Nov 2024 17:25:43 -0800 Subject: [PATCH 018/174] fix(ingest/partitionExecutor): Fetch ready items for non-empty batch when _pending is empty (#11885) --- .../datahub/utilities/partition_executor.py | 22 ++++++------ .../unit/utilities/test_partition_executor.py | 36 ++++++++++++++++--- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/metadata-ingestion/src/datahub/utilities/partition_executor.py b/metadata-ingestion/src/datahub/utilities/partition_executor.py index 0f0e9784464f6d..4d873d8f74bd8e 100644 --- a/metadata-ingestion/src/datahub/utilities/partition_executor.py +++ b/metadata-ingestion/src/datahub/utilities/partition_executor.py @@ -270,7 +270,7 @@ def __init__( self.read_from_pending_interval = read_from_pending_interval assert self.max_workers > 1 - self.state_lock = threading.Lock() + self._state_lock = threading.Lock() self._executor = ThreadPoolExecutor( # We add one here to account for the clearinghouse worker thread. max_workers=max_workers + 1, @@ -323,7 +323,7 @@ def _handle_batch_completion( if item.done_callback: item.done_callback(future) - def _find_ready_items() -> List[_BatchPartitionWorkItem]: + def _find_ready_items(max_to_add: int) -> List[_BatchPartitionWorkItem]: with clearinghouse_state_lock: # First, update the keys in flight. for key in keys_no_longer_in_flight: @@ -336,10 +336,7 @@ def _find_ready_items() -> List[_BatchPartitionWorkItem]: ready: List[_BatchPartitionWorkItem] = [] for item in pending: - if ( - len(ready) < self.max_per_batch - and item.key not in keys_in_flight - ): + if len(ready) < max_to_add and item.key not in keys_in_flight: ready.append(item) else: pending_key_completion.append(item) @@ -347,7 +344,7 @@ def _find_ready_items() -> List[_BatchPartitionWorkItem]: return ready def _build_batch() -> List[_BatchPartitionWorkItem]: - next_batch = _find_ready_items() + next_batch = _find_ready_items(self.max_per_batch) while ( not self._queue_empty_for_shutdown @@ -382,11 +379,12 @@ def _build_batch() -> List[_BatchPartitionWorkItem]: pending_key_completion.append(next_item) else: next_batch.append(next_item) - - if not next_batch: - next_batch = _find_ready_items() except queue.Empty: - if not blocking: + if blocking: + next_batch.extend( + _find_ready_items(self.max_per_batch - len(next_batch)) + ) + else: break return next_batch @@ -458,7 +456,7 @@ def _ensure_clearinghouse_started(self) -> None: f"{self.__class__.__name__} is shutting down; cannot submit new work items." ) - with self.state_lock: + with self._state_lock: # Lazily start the clearinghouse worker. if not self._clearinghouse_started: self._clearinghouse_started = True diff --git a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py index e3a68405e3c0ac..eba79eafce473b 100644 --- a/metadata-ingestion/tests/unit/utilities/test_partition_executor.py +++ b/metadata-ingestion/tests/unit/utilities/test_partition_executor.py @@ -133,9 +133,9 @@ def process_batch(batch): } -@pytest.mark.timeout(10) +@pytest.mark.timeout(5) def test_batch_partition_executor_max_batch_size(): - n = 20 # Exceed max_pending to test for deadlocks when max_pending exceeded + n = 5 batches_processed = [] def process_batch(batch): @@ -147,8 +147,8 @@ def process_batch(batch): max_pending=10, process_batch=process_batch, max_per_batch=2, - min_process_interval=timedelta(seconds=1), - read_from_pending_interval=timedelta(seconds=1), + min_process_interval=timedelta(seconds=0.1), + read_from_pending_interval=timedelta(seconds=0.1), ) as executor: # Submit more tasks than the max_per_batch to test batching limits. for i in range(n): @@ -161,6 +161,34 @@ def process_batch(batch): assert len(batch) <= 2, "Batch size exceeded max_per_batch limit" +@pytest.mark.timeout(10) +def test_batch_partition_executor_deadlock(): + n = 20 # Exceed max_pending to test for deadlocks when max_pending exceeded + batch_size = 2 + batches_processed = [] + + def process_batch(batch): + batches_processed.append(batch) + time.sleep(0.1) # Simulate batch processing time + + with BatchPartitionExecutor( + max_workers=5, + max_pending=2, + process_batch=process_batch, + max_per_batch=batch_size, + min_process_interval=timedelta(seconds=30), + read_from_pending_interval=timedelta(seconds=0.01), + ) as executor: + # Submit more tasks than the max_per_batch to test batching limits. + executor.submit("key3", "key3", "task0") + executor.submit("key3", "key3", "task1") + executor.submit("key1", "key1", "task1") # Populates second batch + for i in range(3, n): + executor.submit("key3", "key3", f"task{i}") + + assert sum(len(batch) for batch in batches_processed) == n + + def test_empty_batch_partition_executor(): # We want to test that even if no submit() calls are made, cleanup works fine. with BatchPartitionExecutor( From 4601aacfadcc49c13571533ca97ee3da442b7026 Mon Sep 17 00:00:00 2001 From: mikeburke24 <42984275+mikeburke24@users.noreply.github.com> Date: Tue, 19 Nov 2024 04:29:37 -0600 Subject: [PATCH 019/174] fix(ingest): upgrade msal (#11883) --- metadata-ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 3152d0290ec227..2469af74b03343 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -245,7 +245,7 @@ # Instead, we put the fix in our PyHive fork, so no thrift pin is needed. } -microsoft_common = {"msal>=1.22.0"} +microsoft_common = {"msal>=1.24.0"} iceberg_common = { # Iceberg Python SDK From a1c783ba3aab69360c700732619d70a08966a0ef Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:58:18 -0600 Subject: [PATCH 020/174] refactor(kafka): reconfigure consumers to allow different config (#11869) --- docker/profiles/docker-compose.gms.yml | 1 - .../kafka/MCLKafkaListenerRegistrar.java | 4 +- .../kafka/MetadataChangeEventsProcessor.java | 4 +- .../MetadataChangeProposalsProcessor.java | 3 +- .../datahub/event/PlatformEventProcessor.java | 4 +- .../config/kafka/ConsumerConfiguration.java | 9 ++ .../config/kafka/KafkaConfiguration.java | 4 + .../src/main/resources/application.yaml | 7 ++ .../kafka/KafkaEventConsumerFactory.java | 88 +++++++++++++++++-- .../test/SchemaRegistryControllerTest.java | 7 +- 10 files changed, 116 insertions(+), 15 deletions(-) diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml index 147bbd35ff6460..824c8024b05d63 100644 --- a/docker/profiles/docker-compose.gms.yml +++ b/docker/profiles/docker-compose.gms.yml @@ -40,7 +40,6 @@ x-kafka-env: &kafka-env # KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 SCHEMA_REGISTRY_TYPE: INTERNAL KAFKA_SCHEMAREGISTRY_URL: http://datahub-gms:8080/schema-registry/api/ - SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET: ${SPRING_KAFKA_CONSUMER_AUTO_OFFSET_RESET:-earliest} x-datahub-quickstart-telemetry-env: &datahub-quickstart-telemetry-env DATAHUB_SERVER_TYPE: ${DATAHUB_SERVER_TYPE:-quickstart} diff --git a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java index fb2880f617d301..c909b0034a9125 100644 --- a/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java +++ b/metadata-jobs/mae-consumer/src/main/java/com/linkedin/metadata/kafka/MCLKafkaListenerRegistrar.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.kafka; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCL_EVENT_CONSUMER_NAME; + import com.linkedin.metadata.kafka.config.MetadataChangeLogProcessorCondition; import com.linkedin.metadata.kafka.hook.MetadataChangeLogHook; import com.linkedin.mxe.Topics; @@ -39,7 +41,7 @@ public class MCLKafkaListenerRegistrar implements InitializingBean { @Autowired private KafkaListenerEndpointRegistry kafkaListenerEndpointRegistry; @Autowired - @Qualifier("kafkaEventConsumer") + @Qualifier(MCL_EVENT_CONSUMER_NAME) private KafkaListenerContainerFactory kafkaListenerContainerFactory; @Value("${METADATA_CHANGE_LOG_KAFKA_CONSUMER_GROUP_ID:generic-mae-consumer-job-client}") diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index 1b3d19915b439e..5d2f6452e69197 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -1,5 +1,7 @@ package com.linkedin.metadata.kafka; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.DEFAULT_EVENT_CONSUMER_NAME; + import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; @@ -60,7 +62,7 @@ public class MetadataChangeEventsProcessor { "${METADATA_CHANGE_EVENT_NAME:${KAFKA_MCE_TOPIC_NAME:" + Topics.METADATA_CHANGE_EVENT + "}}", - containerFactory = "kafkaEventConsumer") + containerFactory = DEFAULT_EVENT_CONSUMER_NAME) @Deprecated public void consume(final ConsumerRecord consumerRecord) { try (Timer.Context i = MetricUtils.timer(this.getClass(), "consume").time()) { diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java index 22c2b4b9c04503..ef87afdef46cb7 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java @@ -4,6 +4,7 @@ import static com.linkedin.metadata.Constants.MDC_CHANGE_TYPE; import static com.linkedin.metadata.Constants.MDC_ENTITY_TYPE; import static com.linkedin.metadata.Constants.MDC_ENTITY_URN; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCP_EVENT_CONSUMER_NAME; import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; @@ -116,7 +117,7 @@ public void registerConsumerThrottle() { @KafkaListener( id = CONSUMER_GROUP_ID_VALUE, topics = "${METADATA_CHANGE_PROPOSAL_TOPIC_NAME:" + Topics.METADATA_CHANGE_PROPOSAL + "}", - containerFactory = "kafkaEventConsumer") + containerFactory = MCP_EVENT_CONSUMER_NAME) public void consume(final ConsumerRecord consumerRecord) { try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "consume").time()) { kafkaLagStats.update(System.currentTimeMillis() - consumerRecord.timestamp()); diff --git a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java index 358a2ac0c2ee33..5d11697bed93d2 100644 --- a/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java +++ b/metadata-jobs/pe-consumer/src/main/java/com/datahub/event/PlatformEventProcessor.java @@ -1,5 +1,7 @@ package com.datahub.event; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.PE_EVENT_CONSUMER_NAME; + import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; @@ -56,7 +58,7 @@ public PlatformEventProcessor( @KafkaListener( id = "${PLATFORM_EVENT_KAFKA_CONSUMER_GROUP_ID:generic-platform-event-job-client}", topics = {"${PLATFORM_EVENT_TOPIC_NAME:" + Topics.PLATFORM_EVENT + "}"}, - containerFactory = "kafkaEventConsumer") + containerFactory = PE_EVENT_CONSUMER_NAME) public void consume(final ConsumerRecord consumerRecord) { try (Timer.Context i = MetricUtils.timer(this.getClass(), "consume").time()) { diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java index 60f3e1b4fef76f..9b476483a2baf8 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/ConsumerConfiguration.java @@ -8,4 +8,13 @@ public class ConsumerConfiguration { private int maxPartitionFetchBytes; private boolean stopOnDeserializationError; private boolean healthCheckEnabled; + + private ConsumerOptions mcp; + private ConsumerOptions mcl; + private ConsumerOptions pe; + + @Data + public static class ConsumerOptions { + private String autoOffsetReset; + } } diff --git a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java index b03aedc1a7b5eb..ae0d3a3bb4647a 100644 --- a/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java +++ b/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/kafka/KafkaConfiguration.java @@ -20,6 +20,10 @@ public class KafkaConfiguration { "spring.deserializer.key.delegate.class"; public static final String VALUE_DESERIALIZER_DELEGATE_CLASS = "spring.deserializer.value.delegate.class"; + public static final String MCP_EVENT_CONSUMER_NAME = "mcpEventConsumer"; + public static final String MCL_EVENT_CONSUMER_NAME = "mclEventConsumer"; + public static final String PE_EVENT_CONSUMER_NAME = "platformEventConsumer"; + public static final String DEFAULT_EVENT_CONSUMER_NAME = "kafkaEventConsumer"; private String bootstrapServers; diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 8010ae187b6c8b..4945b36a251c26 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -289,6 +289,13 @@ kafka: maxPartitionFetchBytes: ${KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES:5242880} # the max bytes consumed per partition stopOnDeserializationError: ${KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR:true} # Stops kafka listener container on deserialization error, allows user to fix problems before moving past problematic offset. If false will log and move forward past the offset healthCheckEnabled: ${KAFKA_CONSUMER_HEALTH_CHECK_ENABLED:true} # Sets the health indicator to down when a message listener container has stopped due to a deserialization failure, will force consumer apps to restart through k8s and docker-compose health mechanisms + mcp: + autoOffsetReset: ${KAFKA_CONSUMER_MCP_AUTO_OFFSET_RESET:earliest} + mcl: + autoOffsetReset: ${KAFKA_CONSUMER_MCL_AUTO_OFFSET_RESET:earliest} + pe: + autoOffsetReset: ${KAFKA_CONSUMER_PE_AUTO_OFFSET_RESET:latest} + schemaRegistry: type: ${SCHEMA_REGISTRY_TYPE:KAFKA} # INTERNAL or KAFKA or AWS_GLUE url: ${KAFKA_SCHEMAREGISTRY_URL:http://localhost:8081} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java index 750af8ec488df3..a1ee4df360b7ec 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/kafka/KafkaEventConsumerFactory.java @@ -1,10 +1,18 @@ package com.linkedin.gms.factory.kafka; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.DEFAULT_EVENT_CONSUMER_NAME; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCL_EVENT_CONSUMER_NAME; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.MCP_EVENT_CONSUMER_NAME; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.PE_EVENT_CONSUMER_NAME; + import com.linkedin.gms.factory.config.ConfigurationProvider; +import com.linkedin.metadata.config.kafka.ConsumerConfiguration; import com.linkedin.metadata.config.kafka.KafkaConfiguration; import java.time.Duration; import java.util.Arrays; +import java.util.HashMap; import java.util.Map; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -23,7 +31,6 @@ @Slf4j @Configuration public class KafkaEventConsumerFactory { - private int kafkaEventConsumerConcurrency; @Bean(name = "kafkaConsumerFactory") @@ -87,15 +94,82 @@ private static Map buildCustomizedProperties( return customizedProperties; } - @Bean(name = "kafkaEventConsumer") + @Bean(name = PE_EVENT_CONSUMER_NAME) + protected KafkaListenerContainerFactory platformEventConsumer( + @Qualifier("kafkaConsumerFactory") + DefaultKafkaConsumerFactory kafkaConsumerFactory, + @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) { + + return buildDefaultKafkaListenerContainerFactory( + PE_EVENT_CONSUMER_NAME, + kafkaConsumerFactory, + configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(), + configurationProvider.getKafka().getConsumer().getPe()); + } + + @Bean(name = MCP_EVENT_CONSUMER_NAME) + protected KafkaListenerContainerFactory mcpEventConsumer( + @Qualifier("kafkaConsumerFactory") + DefaultKafkaConsumerFactory kafkaConsumerFactory, + @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) { + + return buildDefaultKafkaListenerContainerFactory( + MCP_EVENT_CONSUMER_NAME, + kafkaConsumerFactory, + configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(), + configurationProvider.getKafka().getConsumer().getMcp()); + } + + @Bean(name = MCL_EVENT_CONSUMER_NAME) + protected KafkaListenerContainerFactory mclEventConsumer( + @Qualifier("kafkaConsumerFactory") + DefaultKafkaConsumerFactory kafkaConsumerFactory, + @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) { + + return buildDefaultKafkaListenerContainerFactory( + MCL_EVENT_CONSUMER_NAME, + kafkaConsumerFactory, + configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(), + configurationProvider.getKafka().getConsumer().getMcl()); + } + + @Bean(name = DEFAULT_EVENT_CONSUMER_NAME) protected KafkaListenerContainerFactory kafkaEventConsumer( @Qualifier("kafkaConsumerFactory") DefaultKafkaConsumerFactory kafkaConsumerFactory, @Qualifier("configurationProvider") ConfigurationProvider configurationProvider) { + return buildDefaultKafkaListenerContainerFactory( + DEFAULT_EVENT_CONSUMER_NAME, + kafkaConsumerFactory, + configurationProvider.getKafka().getConsumer().isStopOnDeserializationError(), + null); + } + + private KafkaListenerContainerFactory buildDefaultKafkaListenerContainerFactory( + String consumerFactoryName, + DefaultKafkaConsumerFactory kafkaConsumerFactory, + boolean isStopOnDeserializationError, + @Nullable ConsumerConfiguration.ConsumerOptions consumerOptions) { + + final DefaultKafkaConsumerFactory factoryWithOverrides; + if (consumerOptions != null) { + // Copy the base config + Map props = new HashMap<>(kafkaConsumerFactory.getConfigurationProperties()); + // Override just the auto.offset.reset + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, consumerOptions.getAutoOffsetReset()); + factoryWithOverrides = + new DefaultKafkaConsumerFactory<>( + props, + kafkaConsumerFactory.getKeyDeserializer(), + kafkaConsumerFactory.getValueDeserializer()); + } else { + factoryWithOverrides = kafkaConsumerFactory; + } + ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(kafkaConsumerFactory); + factory.setConsumerFactory(factoryWithOverrides); factory.setContainerCustomizer(new ThreadPoolContainerCustomizer()); factory.setConcurrency(kafkaEventConsumerConcurrency); @@ -103,7 +177,7 @@ protected KafkaListenerContainerFactory kafkaEventConsumer( use DefaultErrorHandler (does back-off retry and then logs) rather than stopping the container. Stopping the container prevents lost messages until the error can be examined, disabling this will allow progress, but may lose data */ - if (configurationProvider.getKafka().getConsumer().isStopOnDeserializationError()) { + if (isStopOnDeserializationError) { CommonDelegatingErrorHandler delegatingErrorHandler = new CommonDelegatingErrorHandler(new DefaultErrorHandler()); delegatingErrorHandler.addDelegate( @@ -111,9 +185,9 @@ use DefaultErrorHandler (does back-off retry and then logs) rather than stopping factory.setCommonErrorHandler(delegatingErrorHandler); } log.info( - String.format( - "Event-based KafkaListenerContainerFactory built successfully. Consumer concurrency = %s", - kafkaEventConsumerConcurrency)); + "Event-based {} KafkaListenerContainerFactory built successfully. Consumer concurrency = {}", + consumerFactoryName, + kafkaEventConsumerConcurrency); return factory; } diff --git a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java index 664766f204e460..e8deed00672da7 100644 --- a/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java +++ b/metadata-service/schema-registry-servlet/src/test/java/io/datahubproject/openapi/test/SchemaRegistryControllerTest.java @@ -1,6 +1,7 @@ package io.datahubproject.openapi.test; import static com.linkedin.metadata.Constants.*; +import static com.linkedin.metadata.config.kafka.KafkaConfiguration.DEFAULT_EVENT_CONSUMER_NAME; import static org.testng.Assert.*; import com.linkedin.common.urn.Urn; @@ -199,7 +200,7 @@ public void testPEConsumption() @KafkaListener( id = "test-mcp-consumer", topics = Topics.METADATA_CHANGE_PROPOSAL, - containerFactory = "kafkaEventConsumer", + containerFactory = DEFAULT_EVENT_CONSUMER_NAME, properties = {"auto.offset.reset:earliest"}) public void receiveMCP(ConsumerRecord consumerRecord) { @@ -216,7 +217,7 @@ public void receiveMCP(ConsumerRecord consumerRecord) { @KafkaListener( id = "test-mcl-consumer", topics = Topics.METADATA_CHANGE_LOG_VERSIONED, - containerFactory = "kafkaEventConsumer", + containerFactory = DEFAULT_EVENT_CONSUMER_NAME, properties = {"auto.offset.reset:earliest"}) public void receiveMCL(ConsumerRecord consumerRecord) { @@ -232,7 +233,7 @@ public void receiveMCL(ConsumerRecord consumerRecord) { @KafkaListener( id = "test-pe-consumer", topics = Topics.PLATFORM_EVENT, - containerFactory = "kafkaEventConsumer", + containerFactory = DEFAULT_EVENT_CONSUMER_NAME, properties = {"auto.offset.reset:earliest"}) public void receivePE(ConsumerRecord consumerRecord) { From 77394becd3173b4be112ff0a7f254312a7abe2fc Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:58:42 -0600 Subject: [PATCH 021/174] Update v_0_3_7.md (#11895) --- docs/managed-datahub/release-notes/v_0_3_7.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index 59b7a23b5e5836..19cb04e9f56039 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -11,6 +11,11 @@ Recommended CLI/SDK If you are using an older CLI/SDK version, then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, GitHub Actions, Airflow, in Python SDK somewhere, Java SDK, etc. This is a strong recommendation to upgrade, as we keep on pushing fixes in the CLI, and it helps us support you better. +## Known Issues + +### v0.3.7.3 + * Search page fails to render when filters are applied with a query which returns zero results. + ## Release Changelog --- From d97885749e75098f81d9d34ac78b0e32ee6f2561 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Tue, 19 Nov 2024 10:23:25 -0800 Subject: [PATCH 022/174] docs(structured props): fix a typo in structured property docs (#11887) --- docs/api/tutorials/structured-properties.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/tutorials/structured-properties.md b/docs/api/tutorials/structured-properties.md index 9b18aa922290b4..b606ce9a8e2455 100644 --- a/docs/api/tutorials/structured-properties.md +++ b/docs/api/tutorials/structured-properties.md @@ -66,7 +66,7 @@ mutation createStructuredProperty { qualifiedName:"retentionTime", displayName: "Retention Time", description: "Retention Time is used to figure out how long to retain records in a dataset", - valueType: "urn:li:dataType:number", + valueType: "urn:li:dataType:datahub.number", allowedValues: [ {numberValue: 30, description: "30 days, usually reserved for datasets that are ephemeral and contain pii"}, {numberValue: 90, description:"description: Use this for datasets that drive monthly reporting but contain pii"}, From 2d155ccaa9eb2966295d9c248fdb61a23354f305 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:33:47 -0600 Subject: [PATCH 023/174] feat(mcl-upgrade): implement resume & urn pagination (#11889) --- .../upgrade/system/AbstractMCLStep.java | 131 ++++++++++----- ...ateSchemaFieldsFromSchemaMetadataStep.java | 2 +- .../DatahubUpgradeNonBlockingTest.java | 149 +++++++++++++++++- ...pgradeCliApplicationTestConfiguration.java | 12 +- 4 files changed, 248 insertions(+), 46 deletions(-) diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java index 6c70aee88675c5..cd7947ce3c11aa 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/AbstractMCLStep.java @@ -1,13 +1,12 @@ package com.linkedin.datahub.upgrade.system; -import static com.linkedin.metadata.Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME; - import com.linkedin.common.urn.Urn; import com.linkedin.datahub.upgrade.UpgradeContext; import com.linkedin.datahub.upgrade.UpgradeStep; import com.linkedin.datahub.upgrade.UpgradeStepResult; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.boot.BootstrapStep; import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; @@ -16,10 +15,13 @@ import com.linkedin.metadata.entity.ebean.PartitionedStream; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.upgrade.DataHubUpgradeResult; import com.linkedin.upgrade.DataHubUpgradeState; import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.function.Function; @@ -33,6 +35,8 @@ */ @Slf4j public abstract class AbstractMCLStep implements UpgradeStep { + public static final String LAST_URN_KEY = "lastUrn"; + private final OperationContext opContext; private final EntityService entityService; private final AspectDao aspectDao; @@ -70,10 +74,30 @@ protected Urn getUpgradeIdUrn() { @Override public Function executable() { return (context) -> { + // Resume state + Optional prevResult = + context.upgrade().getUpgradeResult(opContext, getUpgradeIdUrn(), entityService); + String resumeUrn = + prevResult + .filter( + result -> + DataHubUpgradeState.IN_PROGRESS.equals(result.getState()) + && result.getResult() != null + && result.getResult().containsKey(LAST_URN_KEY)) + .map(result -> result.getResult().get(LAST_URN_KEY)) + .orElse(null); + if (resumeUrn != null) { + log.info("{}: Resuming from URN: {}", getUpgradeIdUrn(), resumeUrn); + } // re-using for configuring the sql scan RestoreIndicesArgs args = - new RestoreIndicesArgs().aspectName(getAspectName()).batchSize(batchSize).limit(limit); + new RestoreIndicesArgs() + .aspectName(getAspectName()) + .batchSize(batchSize) + .lastUrn(resumeUrn) + .urnBasedPagination(resumeUrn != null) + .limit(limit); if (getUrnLike() != null) { args = args.urnLike(getUrnLike()); @@ -86,40 +110,62 @@ public Function executable() { batch -> { log.info("Processing batch({}) of size {}.", getAspectName(), batchSize); - List, Boolean>> futures; - + List, SystemAspect>> futures; futures = EntityUtils.toSystemAspectFromEbeanAspects( opContext.getRetrieverContext().get(), batch.collect(Collectors.toList())) .stream() .map( - systemAspect -> - entityService.alwaysProduceMCLAsync( - opContext, - systemAspect.getUrn(), - systemAspect.getUrn().getEntityType(), - getAspectName(), - systemAspect.getAspectSpec(), - null, - systemAspect.getRecordTemplate(), - null, - systemAspect - .getSystemMetadata() - .setRunId(id()) - .setLastObserved(System.currentTimeMillis()), - AuditStampUtils.createDefaultAuditStamp(), - ChangeType.UPSERT)) - .collect(Collectors.toList()); - - futures.forEach( - f -> { - try { - f.getFirst().get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - }); + systemAspect -> { + Pair, Boolean> future = + entityService.alwaysProduceMCLAsync( + opContext, + systemAspect.getUrn(), + systemAspect.getUrn().getEntityType(), + getAspectName(), + systemAspect.getAspectSpec(), + null, + systemAspect.getRecordTemplate(), + null, + systemAspect + .getSystemMetadata() + .setRunId(id()) + .setLastObserved(System.currentTimeMillis()), + AuditStampUtils.createDefaultAuditStamp(), + ChangeType.UPSERT); + return Pair., SystemAspect>of( + future.getFirst(), systemAspect); + }) + .toList(); + + SystemAspect lastAspect = + futures.stream() + .map( + f -> { + try { + f.getFirst().get(); + return f.getSecond(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }) + .reduce((a, b) -> b) + .orElse(null); + + // record progress + if (lastAspect != null) { + log.info( + "{}: Saving state. Last urn:{}", getUpgradeIdUrn(), lastAspect.getUrn()); + context + .upgrade() + .setUpgradeResult( + opContext, + getUpgradeIdUrn(), + entityService, + DataHubUpgradeState.IN_PROGRESS, + Map.of(LAST_URN_KEY, lastAspect.getUrn().toString())); + } if (batchDelayMs > 0) { log.info("Sleeping for {} ms", batchDelayMs); @@ -142,12 +188,23 @@ public Function executable() { @Override /** Returns whether the upgrade should be skipped. */ public boolean skip(UpgradeContext context) { - boolean previouslyRun = - entityService.exists( - opContext, getUpgradeIdUrn(), DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, true); - if (previouslyRun) { - log.info("{} was already run. Skipping.", id()); + Optional prevResult = + context.upgrade().getUpgradeResult(opContext, getUpgradeIdUrn(), entityService); + + boolean previousRunFinal = + prevResult + .filter( + result -> + DataHubUpgradeState.SUCCEEDED.equals(result.getState()) + || DataHubUpgradeState.ABORTED.equals(result.getState())) + .isPresent(); + + if (previousRunFinal) { + log.info( + "{} was already run. State: {} Skipping.", + id(), + prevResult.map(DataHubUpgradeResult::getState)); } - return previouslyRun; + return previousRunFinal; } } diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java index eece83f4ab713e..55bc8edbf6a768 100644 --- a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java +++ b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/schemafield/GenerateSchemaFieldsFromSchemaMetadataStep.java @@ -1,5 +1,6 @@ package com.linkedin.datahub.upgrade.system.schemafield; +import static com.linkedin.datahub.upgrade.system.AbstractMCLStep.LAST_URN_KEY; import static com.linkedin.metadata.Constants.APP_SOURCE; import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; import static com.linkedin.metadata.Constants.SCHEMA_METADATA_ASPECT_NAME; @@ -61,7 +62,6 @@ */ @Slf4j public class GenerateSchemaFieldsFromSchemaMetadataStep implements UpgradeStep { - private static final String LAST_URN_KEY = "lastUrn"; private static final List REQUIRED_ASPECTS = List.of(SCHEMA_METADATA_ASPECT_NAME, STATUS_ASPECT_NAME); diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java index f340e688ad7f77..21bc6b725cba2b 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java @@ -1,14 +1,18 @@ package com.linkedin.datahub.upgrade; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertNotNull; +import com.linkedin.data.template.StringMap; import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager; import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking; import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCPStep; @@ -20,17 +24,30 @@ import com.linkedin.metadata.entity.AspectDao; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.EntityServiceImpl; +import com.linkedin.metadata.entity.ebean.EbeanAspectV2; +import com.linkedin.metadata.entity.ebean.PartitionedStream; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.mxe.Topics; +import com.linkedin.upgrade.DataHubUpgradeResult; +import com.linkedin.upgrade.DataHubUpgradeState; +import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.sql.Timestamp; +import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.inject.Named; +import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @ActiveProfiles("test") @@ -63,7 +80,12 @@ public class DatahubUpgradeNonBlockingTest extends AbstractTestNGSpringContextTe @Autowired private EntityServiceImpl entityService; - @Autowired private OperationContext opContext; + private OperationContext opContext; + + @BeforeClass + public void init() { + opContext = TestOperationContexts.systemContextNoValidate(); + } @Test public void testSystemUpdateNonBlockingInit() { @@ -81,10 +103,13 @@ public void testSystemUpdateNonBlockingInit() { } @Test - public void testReindexDataJobViaNodesCLLPaging() { + public void testReindexDataJobViaNodesCLLPagingArgs() { EntityService mockService = mock(EntityService.class); AspectDao mockAspectDao = mock(AspectDao.class); + PartitionedStream mockStream = mock(PartitionedStream.class); + when(mockStream.partition(anyInt())).thenReturn(Stream.empty()); + when(mockAspectDao.streamAspectBatches(any(RestoreIndicesArgs.class))).thenReturn(mockStream); ReindexDataJobViaNodesCLL cllUpgrade = new ReindexDataJobViaNodesCLL(opContext, mockService, mockAspectDao, true, 10, 0, 0); @@ -102,9 +127,79 @@ public void testReindexDataJobViaNodesCLLPaging() { .batchSize(10) .limit(0) .aspectName("dataJobInputOutput") + .urnBasedPagination(false) + .lastUrn(null) .urnLike("urn:li:dataJob:%"))); } + @Test + public void testReindexDataJobViaNodesCLLResumePaging() throws Exception { + // Mock services + EntityService mockService = mock(EntityService.class); + AspectDao mockAspectDao = mock(AspectDao.class); + + // Create test data + EbeanAspectV2 aspect1 = createMockEbeanAspect("urn:li:dataJob:job1", "dataJobInputOutput"); + EbeanAspectV2 aspect2 = createMockEbeanAspect("urn:li:dataJob:job2", "dataJobInputOutput"); + EbeanAspectV2 aspect3 = createMockEbeanAspect("urn:li:dataJob:job3", "dataJobInputOutput"); + List initialBatch = Arrays.asList(aspect1, aspect2); + List resumeBatch = Arrays.asList(aspect3); + + // Mock the stream for first batch + PartitionedStream initialStream = mock(PartitionedStream.class); + when(initialStream.partition(anyInt())).thenReturn(Stream.of(initialBatch.stream())); + + // Mock the stream for second batch + PartitionedStream resumeStream = mock(PartitionedStream.class); + when(resumeStream.partition(anyInt())).thenReturn(Stream.of(resumeBatch.stream())); + + // Setup the AspectDao using Answer to handle null safely + when(mockAspectDao.streamAspectBatches(any(RestoreIndicesArgs.class))) + .thenAnswer( + invocation -> { + RestoreIndicesArgs args = invocation.getArgument(0); + if (args.lastUrn() == null) { + return initialStream; + } else if ("urn:li:dataJob:job2".equals(args.lastUrn())) { + return resumeStream; + } + return mock(PartitionedStream.class); + }); + + // Mock successful MCL production + when(mockService.alwaysProduceMCLAsync( + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())) + .thenReturn(Pair.of(CompletableFuture.completedFuture(null), true)); + + // Create the upgrade + ReindexDataJobViaNodesCLL cllUpgrade = + new ReindexDataJobViaNodesCLL(opContext, mockService, mockAspectDao, true, 2, 0, 0); + + // Initial Run + cllUpgrade.steps().get(0).executable().apply(createMockInitialUpgrade()); + + // Resumed + cllUpgrade.steps().get(0).executable().apply(createMockResumeUpgrade()); + + // Use ArgumentCaptor to verify the calls + ArgumentCaptor argsCaptor = + ArgumentCaptor.forClass(RestoreIndicesArgs.class); + verify(mockAspectDao, times(2)).streamAspectBatches(argsCaptor.capture()); + + List capturedArgs = argsCaptor.getAllValues(); + + // Verify both the initial and resume calls were made with correct arguments + assertEquals(capturedArgs.get(0).lastUrn(), null); + assertEquals(capturedArgs.get(0).urnBasedPagination(), false); + assertEquals(capturedArgs.get(1).lastUrn(), "urn:li:dataJob:job2"); + assertEquals(capturedArgs.get(1).urnBasedPagination(), true); + + // Verify MCL production was called for each aspect + verify(mockService, times(3)) + .alwaysProduceMCLAsync( + any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()); + } + @Test public void testNonBlockingBootstrapMCP() { List mcpTemplate = @@ -123,4 +218,54 @@ public void testNonBlockingBootstrapMCP() { .map(update -> update.getMcpTemplate().getName()) .collect(Collectors.toSet()))); } + + private UpgradeContext createMockInitialUpgrade() { + // Mock the Upgrade instance + Upgrade mockUpgrade = mock(Upgrade.class); + + // Configure the mock upgrade to return no previous result + when(mockUpgrade.getUpgradeResult(any(), any(), any())).thenReturn(Optional.empty()); + + UpgradeContext mockInitialContext = mock(UpgradeContext.class); + when(mockInitialContext.opContext()).thenReturn(opContext); + when(mockInitialContext.upgrade()).thenReturn(mockUpgrade); + when(mockInitialContext.report()).thenReturn(mock(UpgradeReport.class)); + + return mockInitialContext; + } + + private UpgradeContext createMockResumeUpgrade() { + // Mock the Upgrade instance + Upgrade mockUpgrade = mock(Upgrade.class); + DataHubUpgradeResult mockPrevResult = mock(DataHubUpgradeResult.class); + + // Configure the mock previous result + when(mockPrevResult.getState()).thenReturn(DataHubUpgradeState.IN_PROGRESS); + when(mockPrevResult.getResult()) + .thenReturn(new StringMap(Map.of("lastUrn", "urn:li:dataJob:job2"))); + + // Configure the mock upgrade to return our previous result + when(mockUpgrade.getUpgradeResult(any(), any(), any())).thenReturn(Optional.of(mockPrevResult)); + + UpgradeContext mockResumeContext = mock(UpgradeContext.class); + when(mockResumeContext.opContext()).thenReturn(opContext); + when(mockResumeContext.upgrade()).thenReturn(mockUpgrade); + when(mockResumeContext.report()).thenReturn(mock(UpgradeReport.class)); + + return mockResumeContext; + } + + private static EbeanAspectV2 createMockEbeanAspect(String urn, String aspectName) { + Timestamp now = new Timestamp(System.currentTimeMillis()); + return new EbeanAspectV2( + urn, + aspectName, + 0L, + "{}", // metadata + now, // createdOn + "urn:li:corpuser:testUser", // createdBy + null, // createdFor + null // systemMetadata + ); + } } diff --git a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java index 81d883d8ce36b7..5b7b8756f11fb1 100644 --- a/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java +++ b/datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/UpgradeCliApplicationTestConfiguration.java @@ -19,17 +19,17 @@ @Import(value = {SystemAuthenticationFactory.class}) public class UpgradeCliApplicationTestConfiguration { - @MockBean private UpgradeCli upgradeCli; + @MockBean public UpgradeCli upgradeCli; - @MockBean private Database ebeanServer; + @MockBean public Database ebeanServer; - @MockBean private SearchService searchService; + @MockBean public SearchService searchService; - @MockBean private GraphService graphService; + @MockBean public GraphService graphService; - @MockBean private EntityRegistry entityRegistry; + @MockBean public EntityRegistry entityRegistry; - @MockBean ConfigEntityRegistry configEntityRegistry; + @MockBean public ConfigEntityRegistry configEntityRegistry; @MockBean public EntityIndexBuilders entityIndexBuilders; From 44affd7f8211cb902112156660666b05b5f4dbe6 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 19 Nov 2024 17:24:17 -0500 Subject: [PATCH 024/174] fix(ui) Fix merging siblings schema with mix of v1 & v2 fields (#11837) --- .../src/app/entity/shared/siblingUtils.ts | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/siblingUtils.ts b/datahub-web-react/src/app/entity/shared/siblingUtils.ts index 2f50dc99df191b..aa9e4bcb5e46e1 100644 --- a/datahub-web-react/src/app/entity/shared/siblingUtils.ts +++ b/datahub-web-react/src/app/entity/shared/siblingUtils.ts @@ -5,6 +5,7 @@ import * as QueryString from 'query-string'; import { Dataset, Entity, Maybe, SiblingProperties } from '../../../types.generated'; import { GenericEntityProperties } from './types'; import { useIsShowSeparateSiblingsEnabled } from '../../useAppConfig'; +import { downgradeV2FieldPath } from '../dataset/profile/schema/utils/utils'; export function stripSiblingsFromEntity(entity: any) { return { @@ -55,16 +56,30 @@ const combineMerge = (target, source, options) => { return destination; }; -function convertObjectKeysToLowercase(object: Record) { - return Object.fromEntries(Object.entries(object).map(([key, value]) => [key.toLowerCase(), value])); +// this function is responsible for normalizing object keys to make sure merging on key matches keys appropriately +function normalizeObjectKeys(object: Record, isSchemaField = false) { + return Object.fromEntries( + Object.entries(object).map(([key, value]) => { + let normalizedKey = key.toLowerCase(); + if (isSchemaField) { + normalizedKey = downgradeV2FieldPath(normalizedKey) || normalizedKey; + } + return [normalizedKey, value]; + }), + ); } // use when you want to merge an array of objects by key in the object as opposed to by index of array -const mergeArrayOfObjectsByKey = (destinationArray: any[], sourceArray: any[], key: string) => { - const destination = convertObjectKeysToLowercase(keyBy(destinationArray, key)); - const source = convertObjectKeysToLowercase(keyBy(sourceArray, key)); +const mergeArrayOfObjectsByKey = (destinationArray: any[], sourceArray: any[], key: string, isSchemaField = false) => { + const destination = normalizeObjectKeys(keyBy(destinationArray, key), isSchemaField); + const source = normalizeObjectKeys(keyBy(sourceArray, key), isSchemaField); - return values(merge(destination, source)); + return values( + merge(destination, source, { + arrayMerge: combineMerge, + customMerge, + }), + ); }; const mergeTags = (destinationArray, sourceArray, _options) => { @@ -88,7 +103,7 @@ const mergeOwners = (destinationArray, sourceArray, _options) => { }; const mergeFields = (destinationArray, sourceArray, _options) => { - return mergeArrayOfObjectsByKey(destinationArray, sourceArray, 'fieldPath'); + return mergeArrayOfObjectsByKey(destinationArray, sourceArray, 'fieldPath', true); }; function getArrayMergeFunction(key) { @@ -112,7 +127,7 @@ function getArrayMergeFunction(key) { } } -const customMerge = (isPrimary, key) => { +function customMerge(isPrimary, key) { if (key === 'upstream' || key === 'downstream') { return (_secondary, primary) => primary; } @@ -145,7 +160,7 @@ const customMerge = (isPrimary, key) => { customMerge: customMerge.bind({}, isPrimary), }); }; -}; +} export const getEntitySiblingData = (baseEntity: T): Maybe => { if (!baseEntity) { From 85c8e605be045deb59f7548380b550d12e70c900 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 19 Nov 2024 15:06:16 -0800 Subject: [PATCH 025/174] fix(ingest): consider sql parsing fallback as failure (#11896) --- metadata-ingestion/src/datahub/cli/check_cli.py | 4 +++- .../src/datahub/sql_parsing/sqlglot_lineage.py | 9 +++++++++ .../goldens/test_sqlite_attach_database.json | 12 ++++++++++++ .../tests/unit/sql_parsing/test_sqlglot_lineage.py | 11 +++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_attach_database.json diff --git a/metadata-ingestion/src/datahub/cli/check_cli.py b/metadata-ingestion/src/datahub/cli/check_cli.py index 39ed1b2bfea087..fbe07b64f0e154 100644 --- a/metadata-ingestion/src/datahub/cli/check_cli.py +++ b/metadata-ingestion/src/datahub/cli/check_cli.py @@ -268,7 +268,9 @@ def sql_lineage( ) logger.debug("Sql parsing debug info: %s", lineage.debug_info) - if lineage.debug_info.error: + if lineage.debug_info.table_error: + raise lineage.debug_info.table_error + elif lineage.debug_info.error: logger.debug("Sql parsing error details", exc_info=lineage.debug_info.error) click.echo(lineage.json(indent=4)) diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index b635f8cb47b6d2..506bd1d8c6be40 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -904,6 +904,15 @@ def _sqlglot_lineage_inner( logger.debug("Parsing lineage from sql statement: %s", sql) statement = parse_statement(sql, dialect=dialect) + if isinstance(statement, sqlglot.exp.Command): + # For unsupported syntax, sqlglot will usually fallback to parsing as a Command. + # This is effectively a parsing error, and we won't get any lineage from it. + # See https://github.com/tobymao/sqlglot/commit/3a13fdf4e597a2f0a3f9fc126a129183fe98262f + # and https://github.com/tobymao/sqlglot/pull/2874 + raise UnsupportedStatementTypeError( + f"Got unsupported syntax for statement: {sql}" + ) + original_statement, statement = statement, statement.copy() # logger.debug( # "Formatted sql statement: %s", diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_attach_database.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_attach_database.json new file mode 100644 index 00000000000000..bcf31f6be803a2 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_sqlite_attach_database.json @@ -0,0 +1,12 @@ +{ + "query_type": "UNKNOWN", + "query_type_props": {}, + "query_fingerprint": null, + "in_tables": [], + "out_tables": [], + "column_lineage": null, + "debug_info": { + "confidence": 0.0, + "generalized_statement": null + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index 90cc863d6bd231..170341230205f3 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -1268,3 +1268,14 @@ def test_bigquery_subquery_column_inference() -> None: dialect="bigquery", expected_file=RESOURCE_DIR / "test_bigquery_subquery_column_inference.json", ) + + +def test_sqlite_attach_database() -> None: + assert_sql_result( + """\ +ATTACH DATABASE ':memory:' AS aux1 +""", + dialect="sqlite", + expected_file=RESOURCE_DIR / "test_sqlite_attach_database.json", + allow_table_error=True, + ) From 1f396e87c1c48ad7b8a9996dc94c227ffd53e876 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Wed, 20 Nov 2024 00:13:24 +0100 Subject: [PATCH 026/174] feat(spark): OpenLineage 1.24.2 upgrade (#11830) --- build.gradle | 2 +- entity-registry/build.gradle | 2 ++ .../java/acryl-spark-lineage/README.md | 1 + .../java/acryl-spark-lineage/build.gradle | 35 ++++++++++--------- .../datahub/spark/DatahubSparkListener.java | 8 +++-- .../datahub/spark/conf/SparkConfigParser.java | 2 ++ .../spark/agent/util/PlanUtils.java | 8 ++--- .../spark/agent/util/RddPathUtils.java | 18 ++++++---- .../java/datahub-client/build.gradle | 28 ++++++++++----- .../rest/DatahubHttpRequestRetryStrategy.java | 1 - .../java/datahub/client/rest/RestEmitter.java | 33 +++++++++++------ .../client/rest/RestEmitterConfig.java | 2 ++ .../java/openlineage-converter/build.gradle | 2 +- 13 files changed, 89 insertions(+), 53 deletions(-) diff --git a/build.gradle b/build.gradle index 6e6dadb7ebfa34..9ee756d41e11ef 100644 --- a/build.gradle +++ b/build.gradle @@ -56,7 +56,7 @@ buildscript { ext.hazelcastVersion = '5.3.6' ext.ebeanVersion = '15.5.2' ext.googleJavaFormatVersion = '1.18.1' - ext.openLineageVersion = '1.19.0' + ext.openLineageVersion = '1.24.2' ext.logbackClassicJava8 = '1.2.12' ext.docker_registry = 'acryldata' diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle index 2dedea1f16d99c..22e5b601d39db2 100644 --- a/entity-registry/build.gradle +++ b/entity-registry/build.gradle @@ -25,6 +25,8 @@ dependencies { because("previous versions are vulnerable to CVE-2022-25857") } } + api project(path: ':li-utils') + api project(path: ':li-utils', configuration: "dataTemplate") dataModel project(':li-utils') annotationProcessor externalDependency.lombok diff --git a/metadata-integration/java/acryl-spark-lineage/README.md b/metadata-integration/java/acryl-spark-lineage/README.md index bd0a58b635b483..267e979b0fa073 100644 --- a/metadata-integration/java/acryl-spark-lineage/README.md +++ b/metadata-integration/java/acryl-spark-lineage/README.md @@ -165,6 +165,7 @@ information like tokens. | spark.datahub.rest.server | | http://localhost:8080 | Datahub server url eg: | | spark.datahub.rest.token | | | Authentication token. | | spark.datahub.rest.disable_ssl_verification | | false | Disable SSL certificate validation. Caution: Only use this if you know what you are doing! | +| spark.datahub.rest.disable_chunked_encoding | | false | Disable Chunked Transfer Encoding. In some environment chunked encoding causes issues. With this config option it can be disabled. || | spark.datahub.rest.max_retries | | 0 | Number of times a request retried if failed | | spark.datahub.rest.retry_interval | | 10 | Number of seconds to wait between retries | | spark.datahub.file.filename | | | The file where metadata will be written if file emitter is set | diff --git a/metadata-integration/java/acryl-spark-lineage/build.gradle b/metadata-integration/java/acryl-spark-lineage/build.gradle index 6620c34021ac4a..3f83e5657bbf4d 100644 --- a/metadata-integration/java/acryl-spark-lineage/build.gradle +++ b/metadata-integration/java/acryl-spark-lineage/build.gradle @@ -1,7 +1,7 @@ plugins { id("com.palantir.git-version") apply false } -apply plugin: 'java' +apply plugin: 'java-library' apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' @@ -51,8 +51,8 @@ dependencies { implementation project(':metadata-integration:java:openlineage-converter') - implementation project(path: ':metadata-integration:java:datahub-client', configuration: 'shadow') - implementation project(path: ':metadata-integration:java:openlineage-converter', configuration: 'shadow') + implementation project(path: ':metadata-integration:java:datahub-client') + implementation project(path: ':metadata-integration:java:openlineage-converter') //implementation "io.acryl:datahub-client:0.10.2" implementation "io.openlineage:openlineage-spark_2.12:$openLineageVersion" @@ -91,6 +91,8 @@ shadowJar { zip64 = true archiveClassifier = '' mergeServiceFiles() + project.configurations.implementation.canBeResolved = true + configurations = [project.configurations.implementation] def exclude_modules = project .configurations @@ -106,6 +108,8 @@ shadowJar { exclude(dependency { exclude_modules.contains(it.name) }) + exclude(dependency("org.slf4j::")) + exclude("org/apache/commons/logging/**") } // preventing java multi-release JAR leakage @@ -123,39 +127,36 @@ shadowJar { relocate 'com.sun.activation', 'io.acryl.shaded.com.sun.activation' relocate 'com.sun.codemodel', 'io.acryl.shaded.com.sun.codemodel' relocate 'com.sun.mail', 'io.acryl.shaded.com.sun.mail' - relocate 'com.fasterxml.jackson', 'datahub.spark2.shaded.jackson' - relocate 'org.slf4j', 'datahub.spark2.shaded.org.slf4j' // relocate 'org.apache.hc', 'io.acryl.shaded.http' - relocate 'org.apache.commons.codec', 'datahub.spark2.shaded.o.a.c.codec' - relocate 'org.apache.commons.compress', 'datahub.spark2.shaded.o.a.c.compress' - relocate 'org.apache.commons.lang3', 'datahub.spark2.shaded.o.a.c.lang3' + relocate 'org.apache.commons.codec', 'io.acryl.shaded.org.apache.commons.codec' + relocate 'org.apache.commons.compress', 'io.acryl.shaded.org.apache.commons.compress' + relocate 'org.apache.commons.lang3', 'io.acryl.shaded.org.apache.commons.lang3' relocate 'mozilla', 'datahub.spark2.shaded.mozilla' - relocate 'com.typesafe', 'datahub.spark2.shaded.typesafe' - relocate 'io.opentracing', 'datahub.spark2.shaded.io.opentracing' - relocate 'io.netty', 'datahub.spark2.shaded.io.netty' - relocate 'ch.randelshofer', 'datahub.spark2.shaded.ch.randelshofer' - relocate 'ch.qos', 'datahub.spark2.shaded.ch.qos' + relocate 'com.typesafe', 'io.acryl.shaded.com.typesafe' + relocate 'io.opentracing', 'io.acryl.shaded.io.opentracing' + relocate 'io.netty', 'io.acryl.shaded.io.netty' + relocate 'ch.randelshofer', 'io.acryl.shaded.ch.randelshofer' + relocate 'ch.qos', 'io.acryl.shaded.ch.qos' relocate 'org.springframework', 'io.acryl.shaded.org.springframework' relocate 'com.fasterxml.jackson', 'io.acryl.shaded.jackson' relocate 'org.yaml', 'io.acryl.shaded.org.yaml' // Required for shading snakeyaml relocate 'net.jcip.annotations', 'io.acryl.shaded.annotations' relocate 'javassist', 'io.acryl.shaded.javassist' relocate 'edu.umd.cs.findbugs', 'io.acryl.shaded.findbugs' - relocate 'org.antlr', 'io.acryl.shaded.org.antlr' - relocate 'antlr', 'io.acryl.shaded.antlr' + //relocate 'org.antlr', 'io.acryl.shaded.org.antlr' + //relocate 'antlr', 'io.acryl.shaded.antlr' relocate 'com.google.common', 'io.acryl.shaded.com.google.common' - relocate 'org.apache.commons', 'io.acryl.shaded.org.apache.commons' relocate 'org.reflections', 'io.acryl.shaded.org.reflections' relocate 'st4hidden', 'io.acryl.shaded.st4hidden' relocate 'org.stringtemplate', 'io.acryl.shaded.org.stringtemplate' relocate 'org.abego.treelayout', 'io.acryl.shaded.treelayout' - relocate 'org.slf4j', 'io.acryl.shaded.slf4j' relocate 'javax.annotation', 'io.acryl.shaded.javax.annotation' relocate 'com.github.benmanes.caffeine', 'io.acryl.shaded.com.github.benmanes.caffeine' relocate 'org.checkerframework', 'io.acryl.shaded.org.checkerframework' relocate 'com.google.errorprone', 'io.acryl.shaded.com.google.errorprone' relocate 'com.sun.jna', 'io.acryl.shaded.com.sun.jna' + } checkShadowJar { diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java index ee0938edb50454..b594f6bae954fa 100644 --- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java +++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/DatahubSparkListener.java @@ -120,7 +120,9 @@ public Optional initializeEmitter(Config sparkConf) { boolean disableSslVerification = sparkConf.hasPath(SparkConfigParser.DISABLE_SSL_VERIFICATION_KEY) && sparkConf.getBoolean(SparkConfigParser.DISABLE_SSL_VERIFICATION_KEY); - + boolean disableChunkedEncoding = + sparkConf.hasPath(SparkConfigParser.REST_DISABLE_CHUNKED_ENCODING) + && sparkConf.getBoolean(SparkConfigParser.REST_DISABLE_CHUNKED_ENCODING); int retry_interval_in_sec = sparkConf.hasPath(SparkConfigParser.RETRY_INTERVAL_IN_SEC) ? sparkConf.getInt(SparkConfigParser.RETRY_INTERVAL_IN_SEC) @@ -150,6 +152,7 @@ public Optional initializeEmitter(Config sparkConf) { .disableSslVerification(disableSslVerification) .maxRetries(max_retries) .retryIntervalSec(retry_interval_in_sec) + .disableChunkedEncoding(disableChunkedEncoding) .build(); return Optional.of(new RestDatahubEmitterConfig(restEmitterConf)); case "kafka": @@ -374,7 +377,8 @@ private static void initializeMetrics(OpenLineageConfig openLineageConfig) { String disabledFacets; if (openLineageConfig.getFacetsConfig() != null && openLineageConfig.getFacetsConfig().getDisabledFacets() != null) { - disabledFacets = String.join(";", openLineageConfig.getFacetsConfig().getDisabledFacets()); + disabledFacets = + String.join(";", openLineageConfig.getFacetsConfig().getEffectiveDisabledFacets()); } else { disabledFacets = ""; } diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java index 45ec5365d09b36..3860285083c4bb 100644 --- a/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java +++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/datahub/spark/conf/SparkConfigParser.java @@ -30,6 +30,8 @@ public class SparkConfigParser { public static final String GMS_AUTH_TOKEN = "rest.token"; public static final String FILE_EMITTER_FILE_NAME = "file.filename"; public static final String DISABLE_SSL_VERIFICATION_KEY = "rest.disable_ssl_verification"; + public static final String REST_DISABLE_CHUNKED_ENCODING = "rest.disable_chunked_encoding"; + public static final String MAX_RETRIES = "rest.max_retries"; public static final String RETRY_INTERVAL_IN_SEC = "rest.retry_interval_in_sec"; public static final String KAFKA_MCP_TOPIC = "kafka.mcp_topic"; diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java index d46d741d155b8b..5f87df2a65d6c2 100644 --- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java +++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/PlanUtils.java @@ -5,14 +5,13 @@ package io.openlineage.spark.agent.util; -import static io.openlineage.spark.agent.lifecycle.ExecutionContext.CAMEL_TO_SNAKE_CASE; - import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import datahub.spark.conf.SparkLineageConf; import io.datahubproject.openlineage.dataset.HdfsPathDataset; import io.openlineage.client.OpenLineage; import io.openlineage.spark.agent.Versions; +import io.openlineage.spark.api.naming.NameNormalizer; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -21,7 +20,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.UUID; @@ -186,7 +184,7 @@ public static OpenLineage.ParentRunFacet parentRunFacet( .run(new OpenLineage.ParentRunFacetRunBuilder().runId(parentRunId).build()) .job( new OpenLineage.ParentRunFacetJobBuilder() - .name(parentJob.replaceAll(CAMEL_TO_SNAKE_CASE, "_$1").toLowerCase(Locale.ROOT)) + .name(NameNormalizer.normalize(parentJob)) .namespace(parentJobNamespace) .build()) .build(); @@ -287,8 +285,6 @@ public static boolean safeIsDefinedAt(PartialFunction pfn, Object x) { * @param pfn * @param x * @return - * @param - * @param */ public static List safeApply(PartialFunction> pfn, D x) { try { diff --git a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java index 62005bf15f8505..6ef7403362a909 100644 --- a/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java +++ b/metadata-integration/java/acryl-spark-lineage/src/main/java/io/openlineage/spark/agent/util/RddPathUtils.java @@ -7,6 +7,7 @@ import java.util.Arrays; import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.reflect.FieldUtils; @@ -18,6 +19,7 @@ import org.apache.spark.rdd.MapPartitionsRDD; import org.apache.spark.rdd.ParallelCollectionRDD; import org.apache.spark.rdd.RDD; +import org.apache.spark.sql.execution.datasources.FilePartition; import org.apache.spark.sql.execution.datasources.FileScanRDD; import scala.Tuple2; import scala.collection.immutable.Seq; @@ -90,7 +92,7 @@ public boolean isDefinedAt(Object rdd) { @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") public Stream extract(FileScanRDD rdd) { return ScalaConversionUtils.fromSeq(rdd.filePartitions()).stream() - .flatMap(fp -> Arrays.stream(fp.files())) + .flatMap((FilePartition fp) -> Arrays.stream(fp.files())) .map( f -> { if ("3.4".compareTo(package$.MODULE$.SPARK_VERSION()) <= 0) { @@ -115,11 +117,15 @@ public boolean isDefinedAt(Object rdd) { @Override public Stream extract(ParallelCollectionRDD rdd) { + int SEQ_LIMIT = 1000; + AtomicBoolean loggingDone = new AtomicBoolean(false); try { Object data = FieldUtils.readField(rdd, "data", true); log.debug("ParallelCollectionRDD data: {}", data); - if (data instanceof Seq) { - return ScalaConversionUtils.fromSeq((Seq) data).stream() + if ((data instanceof Seq) && ((Seq) data).head() instanceof Tuple2) { + // exit if the first element is invalid + Seq data_slice = (Seq) ((Seq) data).slice(0, SEQ_LIMIT); + return ScalaConversionUtils.fromSeq(data_slice).stream() .map( el -> { Path path = null; @@ -127,9 +133,9 @@ public Stream extract(ParallelCollectionRDD rdd) { // we're able to extract path path = parentOf(((Tuple2) el)._1.toString()); log.debug("Found input {}", path); - } else { - // Change to debug to silence error - log.debug("unable to extract Path from {}", el.getClass().getCanonicalName()); + } else if (!loggingDone.get()) { + log.warn("unable to extract Path from {}", el.getClass().getCanonicalName()); + loggingDone.set(true); } return path; }) diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index d9087347e1b5c6..1bdc848d0385b1 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -1,6 +1,6 @@ plugins { id("com.palantir.git-version") apply false - id 'java' + id 'java-library' id 'com.github.johnrengelman.shadow' id 'jacoco' id 'signing' @@ -12,11 +12,13 @@ apply from: "../versioning.gradle" import org.apache.tools.ant.filters.ReplaceTokens -jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation +jar { + archiveClassifier = "lib" +} dependencies { - implementation project(':entity-registry') - implementation project(':metadata-integration:java:datahub-event') + api project(':entity-registry') + api project(':metadata-integration:java:datahub-event') implementation(externalDependency.kafkaAvroSerializer) { exclude group: "org.apache.avro" } @@ -33,7 +35,7 @@ dependencies { implementation externalDependency.jacksonDataBind runtimeOnly externalDependency.jna - implementation externalDependency.slf4jApi + api externalDependency.slf4jApi compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok // VisibleForTesting @@ -78,6 +80,11 @@ shadowJar { // https://github.com/johnrengelman/shadow/issues/729 exclude('module-info.class', 'META-INF/versions/**', '**/LICENSE', '**/LICENSE*.txt', '**/NOTICE', '**/NOTICE.txt', 'licenses/**', 'log4j2.*', 'log4j.*') + dependencies { + exclude(dependency("org.slf4j::")) + exclude(dependency("antlr::")) + exclude("org/apache/commons/logging/**") + } mergeServiceFiles() // we relocate namespaces manually, because we want to know exactly which libs we are exposing and why // we can move to automatic relocation using ConfigureShadowRelocation after we get to a good place on these first @@ -88,15 +95,20 @@ shadowJar { relocate 'javassist', 'datahub.shaded.javassist' relocate 'edu.umd.cs.findbugs', 'datahub.shaded.findbugs' relocate 'org.antlr', 'datahub.shaded.org.antlr' - relocate 'antlr', 'datahub.shaded.antlr' + //relocate 'antlr', 'datahub.shaded.antlr' relocate 'com.google.common', 'datahub.shaded.com.google.common' - relocate 'org.apache.commons', 'datahub.shaded.org.apache.commons' + relocate 'org.apache.commons.codec', 'datahub.shaded.org.apache.commons.codec' + relocate 'org.apache.commons.compress', 'datahub.shaded.org.apache.commons.compress' + relocate 'org.apache.commons.lang3', 'datahub.shaded.org.apache.commons.lang3' + relocate 'org.apache.commons.lang', 'datahub.shaded.org.apache.commons.lang' + relocate 'org.apache.commons.cli', 'datahub.shaded.org.apache.commons.cli' + relocate 'org.apache.commons.text', 'datahub.shaded.org.apache.commons.text' + relocate 'org.apache.commons.io', 'datahub.shaded.org.apache.commons.io' relocate 'org.apache.maven', 'datahub.shaded.org.apache.maven' relocate 'org.reflections', 'datahub.shaded.org.reflections' relocate 'st4hidden', 'datahub.shaded.st4hidden' relocate 'org.stringtemplate', 'datahub.shaded.org.stringtemplate' relocate 'org.abego.treelayout', 'datahub.shaded.treelayout' - relocate 'org.slf4j', 'datahub.shaded.slf4j' relocate 'javax.annotation', 'datahub.shaded.javax.annotation' relocate 'com.github.benmanes.caffeine', 'datahub.shaded.com.github.benmanes.caffeine' relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework' diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java index 71a4b93baf48f4..50c0277c98b03b 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/DatahubHttpRequestRetryStrategy.java @@ -48,7 +48,6 @@ public boolean retryRequest( @Override public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) { - log.warn("Retrying request due to error: {}", response); return super.retryRequest(response, execCount, context); } } diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java index e1017372be124b..d70c5baf10879d 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitter.java @@ -1,6 +1,7 @@ package datahub.client.rest; import static com.linkedin.metadata.Constants.*; +import static org.apache.hc.core5.http.HttpHeaders.*; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.StreamReadConstraints; @@ -18,6 +19,7 @@ import datahub.event.UpsertAspectRequest; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -26,6 +28,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import javax.annotation.concurrent.ThreadSafe; @@ -97,17 +100,20 @@ public RestEmitter(RestEmitterConfig config) { this.config = config; HttpAsyncClientBuilder httpClientBuilder = this.config.getAsyncHttpClientBuilder(); httpClientBuilder.setRetryStrategy(new DatahubHttpRequestRetryStrategy()); - - // Override httpClient settings with RestEmitter configs if present - if (config.getTimeoutSec() != null) { - httpClientBuilder.setDefaultRequestConfig( - RequestConfig.custom() - .setConnectionRequestTimeout( - config.getTimeoutSec() * 1000, java.util.concurrent.TimeUnit.MILLISECONDS) - .setResponseTimeout( - config.getTimeoutSec() * 1000, java.util.concurrent.TimeUnit.MILLISECONDS) - .build()); + if ((config.getTimeoutSec() != null) || (config.isDisableChunkedEncoding())) { + RequestConfig.Builder requestConfigBuilder = RequestConfig.custom(); + // Override httpClient settings with RestEmitter configs if present + if (config.getTimeoutSec() != null) { + requestConfigBuilder + .setConnectionRequestTimeout(config.getTimeoutSec() * 1000, TimeUnit.MILLISECONDS) + .setResponseTimeout(config.getTimeoutSec() * 1000, TimeUnit.MILLISECONDS); + } + if (config.isDisableChunkedEncoding()) { + requestConfigBuilder.setContentCompressionEnabled(false); + } + httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); } + PoolingAsyncClientConnectionManagerBuilder poolingAsyncClientConnectionManagerBuilder = PoolingAsyncClientConnectionManagerBuilder.create(); @@ -223,8 +229,13 @@ private Future postGeneric( if (this.config.getToken() != null) { simpleRequestBuilder.setHeader("Authorization", "Bearer " + this.config.getToken()); } + if (this.config.isDisableChunkedEncoding()) { + byte[] payloadBytes = payloadJson.getBytes(StandardCharsets.UTF_8); + simpleRequestBuilder.setBody(payloadBytes, ContentType.APPLICATION_JSON); + } else { + simpleRequestBuilder.setBody(payloadJson, ContentType.APPLICATION_JSON); + } - simpleRequestBuilder.setBody(payloadJson, ContentType.APPLICATION_JSON); AtomicReference responseAtomicReference = new AtomicReference<>(); CountDownLatch responseLatch = new CountDownLatch(1); FutureCallback httpCallback = diff --git a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java index e28ad4ed660f0b..55c11aab0ebf3c 100644 --- a/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java +++ b/metadata-integration/java/datahub-client/src/main/java/datahub/client/rest/RestEmitterConfig.java @@ -30,6 +30,8 @@ public class RestEmitterConfig { Integer timeoutSec; @Builder.Default boolean disableSslVerification = false; + @Builder.Default boolean disableChunkedEncoding = false; + @Builder.Default int maxRetries = 0; @Builder.Default int retryIntervalSec = 10; diff --git a/metadata-integration/java/openlineage-converter/build.gradle b/metadata-integration/java/openlineage-converter/build.gradle index 2e04881ab5ccda..d149104f089b36 100644 --- a/metadata-integration/java/openlineage-converter/build.gradle +++ b/metadata-integration/java/openlineage-converter/build.gradle @@ -1,4 +1,4 @@ -apply plugin: 'java' +apply plugin: 'java-library' apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'maven-publish' From 8638bf974a00cb18c837616ed69b794b90de720f Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:51:58 -0600 Subject: [PATCH 027/174] chore(cleanup): remove unused UrnUtils function (#11897) --- .../com/linkedin/common/urn/UrnUtils.java | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java index 89f0cd8fbc9791..0a2400badfc627 100644 --- a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/UrnUtils.java @@ -27,28 +27,6 @@ public static DatasetUrn toDatasetUrn( new DataPlatformUrn(platformName), datasetName, FabricType.valueOf(origin.toUpperCase())); } - /** - * Convert fabric String to FabricType - * - * @param fabric PROD, CORP, EI, DEV, LIT, PRIME - * @return FabricType - */ - @Nonnull - public static FabricType toFabricType(@Nonnull String fabric) { - switch (fabric.toUpperCase()) { - case "PROD": - return FabricType.PROD; - case "CORP": - return FabricType.CORP; - case "EI": - return FabricType.EI; - case "DEV": - return FabricType.DEV; - default: - throw new IllegalArgumentException("Unsupported Fabric Type: " + fabric); - } - } - public static Urn getUrn(String urnStr) { try { return Urn.createFromString(urnStr); From 524ef8c6d0a07961576a2e69b8c3d7e4313550a7 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Nov 2024 00:29:05 -0800 Subject: [PATCH 028/174] perf(ingest/redshift): limit copy lineage (#11662) Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../ingestion/source/redshift/query.py | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py index b18b526ef30fce..71a20890d35e88 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/query.py @@ -9,6 +9,8 @@ # We use 290 instead instead of the standard 320, because escape characters can add to the length. _QUERY_SEQUENCE_LIMIT = 290 +_MAX_COPY_ENTRIES_PER_TABLE = 20 + class RedshiftCommonQuery: CREATE_TEMP_TABLE_CLAUSE = "create temp table" @@ -293,28 +295,37 @@ def alter_table_rename_query( def list_copy_commands_sql( db_name: str, start_time: datetime, end_time: datetime ) -> str: - return """ - select - distinct - "schema" as target_schema, - "table" as target_table, - c.file_name as filename - from - SYS_QUERY_DETAIL as si - join SYS_LOAD_DETAIL as c on - si.query_id = c.query_id - join SVV_TABLE_INFO sti on - sti.table_id = si.table_id - where - database = '{db_name}' - and si.start_time >= '{start_time}' - and si.start_time < '{end_time}' - order by target_schema, target_table, si.start_time asc - """.format( + return """\ +SELECT DISTINCT + target_schema, + target_table, + filename +FROM ( + SELECT + sti."schema" AS target_schema, + sti."table" AS target_table, + c.file_name AS filename, + ROW_NUMBER() OVER ( + PARTITION BY sti."schema", sti."table" + ORDER BY si.start_time DESC + ) AS rn + FROM + SYS_QUERY_DETAIL AS si + JOIN SYS_LOAD_DETAIL AS c ON si.query_id = c.query_id + JOIN SVV_TABLE_INFO sti ON sti.table_id = si.table_id + WHERE + sti.database = '{db_name}' + AND si.start_time >= '{start_time}' + AND si.start_time < '{end_time}' +) subquery +WHERE rn <= {_MAX_COPY_ENTRIES_PER_TABLE} +ORDER BY target_schema, target_table, filename +""".format( # We need the original database name for filtering db_name=db_name, start_time=start_time.strftime(redshift_datetime_format), end_time=end_time.strftime(redshift_datetime_format), + _MAX_COPY_ENTRIES_PER_TABLE=_MAX_COPY_ENTRIES_PER_TABLE, ) @staticmethod From 05d362a94e140d508019d686ccf29b010db5e40d Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 20 Nov 2024 20:47:39 +0530 Subject: [PATCH 029/174] fix(ingest): add error handling (#11905) --- .../src/datahub/ingestion/source/gc/dataprocess_cleanup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 80f7b7a9f4480c..130f2c9c2e12fc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -401,7 +401,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: total_runs=job.get("entity").get("runs").get("total"), ) if datajob_entity.total_runs > 0: - self.delete_dpi_from_datajobs(datajob_entity) + try: + self.delete_dpi_from_datajobs(datajob_entity) + except Exception as e: + logger.error(f"While trying to delete {datajob_entity} got {e}") if ( datajob_entity.total_runs == 0 and self.config.delete_empty_data_jobs From 310f559c6d3b66cbef3620742544624539fc2ebd Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:27:57 -0600 Subject: [PATCH 030/174] chore(docs): Update restli-overview.md (#11908) --- docs/api/restli/restli-overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api/restli/restli-overview.md b/docs/api/restli/restli-overview.md index 22b913d9a25df4..3e9ab00b522670 100644 --- a/docs/api/restli/restli-overview.md +++ b/docs/api/restli/restli-overview.md @@ -1156,7 +1156,7 @@ curl -X POST 'http://localhost:8080/entities?action=search' \ "and": [ { "field": "title", - "value": "Baz Chart 1", + "values": ["Baz Chart 1"], "condition": "EQUAL" } ] @@ -1261,7 +1261,7 @@ curl -X POST 'http://localhost:8080/entities?action=autocomplete' \ "and": [ { "field": "tool", - "value": "looker", + "values": ["looker"], "condition": "EQUAL" } ] From 07ccdc5cc8754f0f4bec7d5769c9ed5d7406673c Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Thu, 21 Nov 2024 02:53:22 +0900 Subject: [PATCH 031/174] docs: add hudi to integrations (#11901) --- docs-website/filterTagIndexes.json | 11 +++++++++++ docs-website/static/img/logos/platforms/hudi.png | Bin 0 -> 7238 bytes 2 files changed, 11 insertions(+) create mode 100644 docs-website/static/img/logos/platforms/hudi.png diff --git a/docs-website/filterTagIndexes.json b/docs-website/filterTagIndexes.json index 2309593b2c3b9f..b269f23cccd667 100644 --- a/docs-website/filterTagIndexes.json +++ b/docs-website/filterTagIndexes.json @@ -231,6 +231,17 @@ "Features": "Stateful Ingestion, UI Ingestion, Status Aspect" } }, + { + "Path": "https://hudi.apache.org/docs/syncing_datahub/", + "imgPath": "img/logos/platforms/hudi.png", + "Title": "Apache Hudi", + "Description": "Apache Hudi is an open-source data lake framework that provides ACID transactions, efficient upserts, time travel queries, and incremental data processing for large-scale datasets.", + "tags": { + "Platform Type": "Datastore", + "Connection Type": "Pull", + "Features": "" + } + }, { "Path": "docs/generated/ingestion/sources/iceberg", "imgPath": "img/logos/platforms/iceberg.png", diff --git a/docs-website/static/img/logos/platforms/hudi.png b/docs-website/static/img/logos/platforms/hudi.png new file mode 100644 index 0000000000000000000000000000000000000000..c5e79bcc86ce3463bda6aa4e73c641d5875e5fb9 GIT binary patch literal 7238 zcmV-M9J%9(P)00CAA0{{R3sNA1j00012P)t-s05)R) zG+_WAK>!^-06bv;N?ic3>ql@*05MHq=+05xU+gSh}kcK`zf0Ilc%v+e*+e*iaU09=sw+qj+p000?uQchFR@;)RF2KNBi z<-cfArjjVfis5Gb{`{GG5sB-n000|>NklKSpp8F?3&zioMmFNCZF+Hn#Wzuv1B>k#FAtqp zJ>JHCo#*kg&_&T3+1Ri1NL~}Vfq(b?x|fGuHodv|dD_>6&WnDH`MJthgf2SS_ZGPO z0RKYJ*ZFrpU-zQWD`|nc^z*c@1pUz2fIf2fecG3So;AHS^L^SEfnHXR4KNyBi^x~IAkbQ1qAXPx^{El64XQPd+Z z_06D{1pdA0_)@VToBEH3z9sYnLD5!}%Wg&He?0XqV|r;H`dmOi@M{7wNImpXeIw{u z(MbE%nwS;(^_wML>Kj2919JP$y_-YH@0CK6d#P^%T_NuY>a~Ic3I#bz-Ph`Fp>u** zehKIYz0gOg{~0C(0SQal8Sjz zf?I!H>h7QuB44P`pM-vo>Yh3^be>fW8S>ed`;^dqt?mfAC@Bhaean4H=)P8W1Dz|+ zPKSJU4gLJOO)Tb}qq?U~2wkW$jS_iZWeeYYYAsLJzE-D&F1n^CwS@|OZYzzC>ZH(> zdnIN*-*O*px);>^^HH4+I;WQTVue1tXHHgL>QvB0S@hIaEio5!P2ay`RQH3i_Mw+l zW6q#8^tsgCFSZdv7bUrCUu?M_7Pry(#WoV?Wb0|LHT3Cb+%Kv7sBRTH&ngGZtHr&l zi=*|vEoU7%?_4lnYPlbDll4BTTZUd1^x<4rp-*+)4WqjFG>5=;%e*g^&l%Fy4sfaS zH1g1O%qthjmn(Fw&=DgvB1}^5p=%T-T`+GnCr@ab1mIdds#zX+=u5K5+~~Yn4(NKJ zU!|^X%PFBV_E}O#Lg$&2aIR^&A7GneVNzE@XCEeY9CTHxxnZse=n$dbrLGtHhe@67 zk-0XY19!heMrcN%GnPai`nJq96*^Gppj!PRbR~7+p-;9wT`|`MbOEgz^&NPA7W!vJ zbvSh4D2LNl=!jG5^!wFAcC`ELHx7kcT6xvu4Yz?~+) z9Mwhr038$6g`!<2Nqh`~HfWWd2KzdFeGPvMAFKZt1i{xaW{UmJWY5T%4~IBQivfLp zGHxcSFHFll3LRdn^EzR{74zx*!L*%U28BOP2J(AZ&4WQVKWU(;x)PHPRQki7ao0vH zNL}#y2Xv^^IcpbD&%6LfmZtld>i|JFJ=0lm1nnE80A0&0Saf}B* zH!Rj3uTunm1zG7Ya1dCpb8gE`Kz|ZCgRZMb$I}|yoI3!z(E_^saRUc14+I2sfxZ)H zb};ry=uoK(o}AWEZ%04R_Y2)5iIXaQZqg%~miq719VPWJSP#8zzm(jgJwi7L=qhDD2z;Gi0hn`>W8^2J`jpVY zQjgflinVdF1$G^Dd{;&mQ`wmQ%QTSdn9Q2NQzJCdl+b~-y5PAT=0o+q!Bi⋙2cA zCw~X6LYKF#tr7atbgXJDy_(H~0EfYqqVe6Z*sm&8*OYQjc<*(A)NyhYqnbo6fyjsJXiJ0e!{t z?r*8j3LPkQUbiQ^7T50DHEhwjBVGH_hQm^>tk72o{b>e%LFh24i!`UZc%LWhme zM0snUx=QFGu`P65Tc*%~n=b^GUBpAzea%45qtq9L4vXsCX3_S5-HF(lrJlQ%`MiIJ z1@yHSH9wv$7qVr8NIi^CB-ELM$UYAp)pOS~pDhI>D4^rZ$N9lnjnF|+UDz$!9tUjb zC-B+4(CspxJt77c(AQt?cRw+9Iiimpc>RxLzi7Kh_E2mC9d$u&0pd0-)urQpZ3_J! z)wM#0Nj*KDs4O6E$Fue2ngnWf(swi+?5##l#x1R@hfEJ0@YKVE4Em|fNwavix(_0& z49acdYZky+#L~k4nzPg+z0e_27d&t2ptq!1JX_uHg!<>n8W0-Lg^i;M&mDMx&_Pls zD%$S-Iqn!bBBD1L*bzPZ_XhaF2s;~W2C0V-p@T+fQg)&=!rG0kjOa~Qg1^^Or43p` zw@{-$M)f}+<`AhzNlOX6jqAuIbhyx=MfPO@r}_2CP!DEl_oU4K{rSh@*oxFu6%de0 z{S;HBb8MZ^s}l3getlhNKwq=G_Iqa=lsQ1^lttUMDN93r3OWbR%QPtFzr1t>&IYV) zlWzVd$79T3Qr9}yke#ThC!sUw&}S?=v#@PKad;1l`x`#U&-scWNp#M+A<2MdOyC(a+X zb_;ev550p{YX&W&tEUF8q1zVvf8T@D6?IrHr<1zKkAX@ORQw-Kn91|H9{yDSNQUEQ zf}pk4oDdbU$qbQBO z-`JEx1aw|n7W!4{RuXvqORmUAO7ls)j=SBt3T8O0P-SL6#c-woro88cMWKI<(7(p2 z4JdUh&ss@dDgM~*rCw)G{c%Z_#a$f<;&X+4p<|?O@o;D!g>_D@R|)=G5(Nc4J1$l7;aPP!^c?rL=b-?M>)*Yjq5FmD`0i zYfV>QIF-_2xpGySe@9fM*YyNKza;RnTd7~UM=3LF-UKz3(n0Q2c4E!%jfm)t{fLf~ zI>D6>JT)rf+M}%(kePI5)(9P!^^pku)|Rt0LW8%+3_D9npo6^oiinP5PLaC1uhpk> z`I-eLg`2Ps9aa@L&V+8c8TTM{Oaf0bmde8%oyoU>oi~sfc=y5$bL(CAN9bRO1U{sB zrK1#Gx3@vftwRTmE6B_ZbL(AqL#dNZpsV@~EA}?F@xw55NSD>?3mw;%BilwQR36`M z(1m9|LptOUzJt)Q2|QKiD#AQ)-;ZVJY}r*BM=NwptsYbE7?0|s^RUi-u6qK+<*Hum zq$Cvj9uoLls|Fd?S37uy4(XYA3!(2Jfxo(V5jT_#1VZj3H+sa%TfKg zcCCYF$SU1_Oj;cXuK5 zi`3b+wfe1Mk5Q|v+!A!q24y$i-HFimB=vzhreI%}!82TIX4Oji9+uaPqk2Mox|^bY zbmidLV;&~~-rWUsPBIxcoxpE2LNl#(cwApKQ1J*|Kc*s`gmjBS?-zx>*|gkDa}ML- z*UUt-$9|mt8Fns0y69Xn-)1m&p(Pt2ayelGI^a<^odf1XBd{+Zb-@wKFDo{#vxkhC za@-yBtJ|E6(?Lfi@Z?e-dUFm2;TbA)Cu z_q48lUtbTXrOq&&cL#Z#YoQ~j{Z@ho4Glw#0gm z>D3(2)mDIaynV-ul+oE1~aTYGxauk5WJG zD5?*~iUS>dEY?5=^k&Ex(>D_OP3rMpYW3kRR~YryB$c}-a&mj6Zzc5ICGcZA7Pzj? z*C<`;lXw26Lf_T2-1TA9usNDzxCZ*>W%>rqxIIf9crTF2@k0pc)Jo@Wm)EAEdb}s8 z1Bapw7JP5ev#onGn+kn5n^=G3av-V4DJ68`St)Mw%s4#suU$>coy)cuR;mkQ==w)a zyraqGWZY{+b>MBTc>j)o-jHrmwmC5KZDc!?I)jbClR|%Iq3=-Y2bk3H{(Ub>7lqt1 zqVF?-KY*e-ev8vyV!GQA*jGv5Vb}ekPY?aSy=&1(jT4N!WViwv9o;uyUvj4dTu^$b!?6?0^| ze!WHapF%T9VI2}l`ZhDlu<36Sh%QhvGiRJ5(+LtEV`)Ka<}~*apgQuUcP{C5YWxiiGFtAGeHi#Zt0D>2h+Jk z6oQB@FjhAT*x$eX{uCb(1(QZj5Yex~SV2@r_IJ-n{U(71=1}Ox6P@~hK+G~ud zhcJ>!4J#!^ms)ItB=COn$Q#oE-8)BiM*oN+xMZIj&h%RYj>=|?< zKjZcGyEQv}yd*oLc}`QEjOr64;B^DFuNofE83J`9p^h$l;M8^t42|^@Qe-|AZTd+R z!d!Rnfm2kcpN{YEHSrN2xBhfwb|5+EuUE#+z=21Mx|jRzHFJi(qW&(K?+_DT-}C+p_gh^4x(A z1kALaoRMkK#R+2#;>r2wDEtNj)23%cy6zIwa)^9p7sPZVO+tJ8qP%3B%rTM%kw?}+ zOrQL)CL@>V!Xz|E&BZ5Bzdz2yjR9#%T{6*SspXKSRK|(Y^ zphM@fX*!~B1QXrQs&k*&rkw7({Z>JsBknt0CRV?HN+!B2s}96$xeD>#v9ne`euYHv^0A*~H3AbtF5}27->wgZN2> z=z^&31l`(D&@oFJ-{>+6i!SQO2lqs|V9>WeP{JhEl@nY?mnxxgZ|~utWBdC@TF}9x@nx=dkoV-Ou`$S=8}w)rIu?e%T!v0l@t*b)|q78g{kF$Dx394 z6Fh;gF(!TniYewtbP08J?H0c0DgyK^m=_NlKqFom7w^Dx1nWSHOI}25?`Xbk6V%r> zN22>U@IF@ED1`zAoW>X^f%c4}L&CP%l-H(k;7ya2t%9&}OQ1uW9jL z1Ah&~^15x*Fz$dv&h8>X|AjUjd2aACwdj1p*!vc>6YhE-On1xP+epw2N_Ee{a`PoR z6V)Gc4=iNqwjvu7svBAqY zdrj07nm${dm)Rrk2I*9e1|6rm$LJwFBV}ByRY&)~q8Cm3z;^>2-jfFn4-Cr19^y9i zKR7SJ%p?+;;S;z!2)@eeRgE2xHv`=G z%*;=~nKSbzx>u|YTXp1A9%O}`>=5B>1A>Z~NnoR6b-38pSWAI<-WUqdjYs~>>CDYd z0y|x-t}7Cv2cFgxl|XY*eb&jiP_-PyJzbN4o)LlWk)io;G5zgh2}Ea^7)xW+15aS= zOU8bpyL%c$XPGUhathGBocA%%QD4rjmXkgl@|CeVbW}EFdy4~fYzB=R^I1w@rz*B- ziv2zGDDD1Wox-3m#-g*-)sbq7bCrq^bmQKa-G}+iSTHkzS`Mk35+ADU0^J#31LBz_ z5uL%RBeEE`^G*o5lchlW>SA2?jss6OTMlSha@+LXpuR2u^zE;4$be$YvWQMkb*(6p za`{1TgYrF#!ff-V6C*l09f8RuV@7IB#Kh)MY@4u2JI(1r( zHfZb5)86T~P|!`Mll24YWm;Y`?t$tVQC*$Ws9&X%O#svH^)H`qQ2A(yF?yI<&bWg% z%L=)z8)E`Mzq#-m>}a+n6a6wdGH~x-Xr@sxG_P!ux9KSHO$T%^t+HUd9 z2lVqS3Gg(&MH8JkEjQe#PO-{I`{D!o0rkzF4c3ruel|-cx@QJnJB>P9&}K8GO$_J< z1v_}w>e7i$oPlpjHW%)BFrgRdN4u^;Uw`35Cyv#1#z_X<5S@15R6&}4w_|BQ4$#d~1v8uy8lx$VdOl>%`9Poja7Z2v zyyI2dQAX#Zoh}@pr$cm72fnd{8JXB-2k$lSanB)YITEX0UI%s#h&KAU0bqx*P{Ss^_7^ z=lo~XMBgMwbULf9n~3>=3Lhm$3wk1T^d1ME#Hxq)JlLJ0IKySN=)P2kI@?B@SWRfq zhMRMG(Di$gMEAAohPYAPW`2{uX+ckw=wuGO36V$UKIWVp^h7N>Wm-;UA<_Bvg`lTP zbSeo=K%=^+Ij06a8D$)!FxGdvm@U0d;Ufnb>WCXH)#oGn5gd6sE+6*m_biR0?VnE^Jru&n?TIzKtB_GbGvOiH+=p1@9!S; ziEo{UucSI$X$v#w#Gt>u{k*m9uD-0_+1*wD##dJ5a+W>akGJ!O)AenS*Wq?q6@>#2 z25o@+afhkIQ_r*h(>(&fsVl8Fu zoELOc9I4*B)a{eHeyRHJ2d2+;p+h9cGVsn!tUVN1j1bUuae#eb6~{{*zT@ys#p$c_ z6L)v@l^ytUEMM@f>U^up``huleR%xz@bLOX2XTDb)L32Jc{pFbQ;}E^X5{yM4;San?w$RDX*=WHeVE2-Z9LiA-~ZLPLT z1U)gL4`<+&?TgO0O9VYRq7PF2X}Xv#TWyvIy3wf-eUR#p)5UDoP+~3#^z?{6NcHBi zc20`9AkY&e`XJRi(`Tw3G3J6mS6PayUoCe(61r-MAr}OCl0+Y(x@oEO60$*0ljuWK zzXolkGTX(1o+!}=sBVh#JZqrfc(J8`o}%kM;=ms(ZI;2D3j*CFOY|YD-q^tA#2GPN z)n&4v_twr-Ngka`1HDL>=>5euQ|$K9IoHdbtrq>|RPWoxwJOYZ(V!!&n)w3B8y}n$7YuCB&hE~h7c4O=Mhh6@@zSWr2K-a3dE!Fh2M4i|E z!Cn8vLz~3$nd^gVQ$_c7ebjesmoy*9!W&R?t__*R_Jau0^i@0epvg Ubr=SfnE(I)07*qoM6N<$f;07L9smFU literal 0 HcmV?d00001 From 218c05915268015b825737fb103836a0ab679939 Mon Sep 17 00:00:00 2001 From: kanavnarula <33712591+kanavnarula@users.noreply.github.com> Date: Thu, 21 Nov 2024 01:00:21 +0530 Subject: [PATCH 032/174] feat(ui): Display username while removing the user from the group (#11706) --- .../src/app/entity/group/GroupMembers.tsx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/datahub-web-react/src/app/entity/group/GroupMembers.tsx b/datahub-web-react/src/app/entity/group/GroupMembers.tsx index 147c3f8030d0e0..28e81b438d4cb4 100644 --- a/datahub-web-react/src/app/entity/group/GroupMembers.tsx +++ b/datahub-web-react/src/app/entity/group/GroupMembers.tsx @@ -137,12 +137,13 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM }, 3000); }; - const onRemoveMember = (memberUrn: string) => { + const onRemoveMember = (memberEntity: CorpUser) => { + const memberName = entityRegistry.getDisplayName(EntityType.CorpUser, memberEntity); Modal.confirm({ title: `Confirm Group Member Removal`, - content: `Are you sure you want to remove this user from the group?`, + content: `Are you sure you want to remove ${memberName} user from the group?`, onOk() { - removeGroupMember(memberUrn); + removeGroupMember(memberEntity?.urn); }, onCancel() {}, okText: 'Yes', @@ -155,7 +156,7 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM const total = relationships?.total || 0; const groupMembers = relationships?.relationships?.map((rel) => rel.entity as CorpUser) || []; - const getItems = (urnID: string): MenuProps['items'] => { + const getItems = (userEntity: CorpUser): MenuProps['items'] => { return [ { key: 'make', @@ -169,7 +170,7 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM { key: 'remove', disabled: isExternalGroup, - onClick: () => onRemoveMember(urnID), + onClick: () => onRemoveMember(userEntity), label: ( Remove from Group @@ -210,7 +211,7 @@ export default function GroupMembers({ urn, pageSize, isExternalGroup, onChangeM - + From 3f267afc9743d341c35dc1032de0fc0f53655203 Mon Sep 17 00:00:00 2001 From: sid-acryl <155424659+sid-acryl@users.noreply.github.com> Date: Thu, 21 Nov 2024 01:01:54 +0530 Subject: [PATCH 033/174] fix(ingest/powerbi): m-query fixes (#11906) Co-authored-by: Harshal Sheth Co-authored-by: Aseem Bansal --- .../powerbi/m_query/native_sql_parser.py | 20 +++- .../source/powerbi/m_query/resolver.py | 94 ++++++++++--------- .../powerbi/test_native_sql_parser.py | 10 ++ 3 files changed, 78 insertions(+), 46 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py index 61b1164825257e..63a6073c90a1a9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py @@ -72,10 +72,24 @@ def get_tables(native_query: str) -> List[str]: def remove_drop_statement(query: str) -> str: # Certain PowerBI M-Queries contain a combination of DROP and SELECT statements within SQL, causing SQLParser to fail on these queries. # Therefore, these occurrences are being removed. - # Regular expression to match patterns like "DROP TABLE IF EXISTS #;" - pattern = r"DROP TABLE IF EXISTS #\w+;?" - return re.sub(pattern, "", query) + patterns = [ + # Regular expression to match patterns like: + # "DROP TABLE IF EXISTS #;" + # "DROP TABLE IF EXISTS #, , ...;" + # "DROP TABLE IF EXISTS #, , ...\n" + r"DROP\s+TABLE\s+IF\s+EXISTS\s+(?:#?\w+(?:,\s*#?\w+)*)[;\n]", + ] + + new_query = query + + for pattern in patterns: + new_query = re.sub(pattern, "", new_query, flags=re.IGNORECASE) + + # Remove extra spaces caused by consecutive replacements + new_query = re.sub(r"\s+", " ", new_query).strip() + + return new_query def parse_custom_sql( diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py index 9eafde2f75ecdd..32de95d6bd015e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py @@ -83,6 +83,16 @@ def urn_creator( ) +def get_next_item(items: List[str], item: str) -> Optional[str]: + if item in items: + try: + index = items.index(item) + return items[index + 1] + except IndexError: + logger.debug(f'item:"{item}", not found in item-list: {items}') + return None + + class AbstractDataPlatformTableCreator(ABC): """ Base class to share common functionalities among different dataplatform for M-Query parsing. @@ -675,7 +685,7 @@ def two_level_access_pattern( data_access_func_detail.arg_list ) if server is None or db_name is None: - return Lineage.empty() # Return empty list + return Lineage.empty() # Return an empty list schema_name: str = cast( IdentifierAccessor, data_access_func_detail.identifier_accessor @@ -782,32 +792,38 @@ def create_lineage( ), ) - if len(arguments) == 2: - # It is a regular case of MS-SQL - logger.debug("Handling with regular case") - return self.two_level_access_pattern(data_access_func_detail) - - if len(arguments) >= 4 and arguments[2] != "Query": - logger.debug("Unsupported case is found. Second index is not the Query") - return Lineage.empty() + server, database = self.get_db_detail_from_argument( + data_access_func_detail.arg_list + ) + if server is None or database is None: + return Lineage.empty() # Return an empty list + + assert server + assert database # to silent the lint + + query: Optional[str] = get_next_item(arguments, "Query") + if query: + if self.config.enable_advance_lineage_sql_construct is False: + # Use previous parser to generate URN to keep backward compatibility + return Lineage( + upstreams=self.create_urn_using_old_parser( + query=query, + db_name=database, + server=server, + ), + column_lineage=[], + ) - if self.config.enable_advance_lineage_sql_construct is False: - # Use previous parser to generate URN to keep backward compatibility - return Lineage( - upstreams=self.create_urn_using_old_parser( - query=arguments[3], - db_name=arguments[1], - server=arguments[0], - ), - column_lineage=[], + return self.parse_custom_sql( + query=query, + database=database, + server=server, + schema=MSSqlDataPlatformTableCreator.DEFAULT_SCHEMA, ) - return self.parse_custom_sql( - query=arguments[3], - database=arguments[1], - server=arguments[0], - schema=MSSqlDataPlatformTableCreator.DEFAULT_SCHEMA, - ) + # It is a regular case of MS-SQL + logger.debug("Handling with regular case") + return self.two_level_access_pattern(data_access_func_detail) class OracleDataPlatformTableCreator(AbstractDataPlatformTableCreator): @@ -1154,27 +1170,19 @@ def get_db_name(self, data_access_tokens: List[str]) -> Optional[str]: != SupportedDataPlatform.DatabricksMultiCloud_SQL.value.powerbi_data_platform_name ): return None - try: - if "Database" in data_access_tokens: - index = data_access_tokens.index("Database") - if data_access_tokens[index + 1] != Constant.M_QUERY_NULL: - # Database name is explicitly set in argument - return data_access_tokens[index + 1] - if "Name" in data_access_tokens: - index = data_access_tokens.index("Name") - # Next element is value of the Name. It is a database name - return data_access_tokens[index + 1] + database: Optional[str] = get_next_item(data_access_tokens, "Database") - if "Catalog" in data_access_tokens: - index = data_access_tokens.index("Catalog") - # Next element is value of the Catalog. In Databricks Catalog can also be used in place of a database. - return data_access_tokens[index + 1] - - except IndexError as e: - logger.debug("Database name is not available", exc_info=e) - - return None + if ( + database and database != Constant.M_QUERY_NULL + ): # database name is explicitly set + return database + + return get_next_item( # database name is set in Name argument + data_access_tokens, "Name" + ) or get_next_item( # If both above arguments are not available, then try Catalog + data_access_tokens, "Catalog" + ) def create_lineage( self, data_access_func_detail: DataAccessFunctionDetail diff --git a/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py b/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py index 53e184515c1d8c..887f7fe4d6f44a 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py +++ b/metadata-ingestion/tests/integration/powerbi/test_native_sql_parser.py @@ -19,3 +19,13 @@ def test_simple_from(): assert len(tables) == 1 assert tables[0] == "OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4" + + +def test_drop_statement(): + expected: str = "SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4" + + query: str = "DROP TABLE IF EXISTS #table1; DROP TABLE IF EXISTS #table1,#table2; DROP TABLE IF EXISTS table1; DROP TABLE IF EXISTS table1, #table2;SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4" + + actual: str = native_sql_parser.remove_drop_statement(query) + + assert actual == expected From 2a37483b49fb6d2b76defd99e1678add6e9d4527 Mon Sep 17 00:00:00 2001 From: Meenakshi Kamalaseshan Radha <62914384+mkamalas@users.noreply.github.com> Date: Thu, 21 Nov 2024 01:37:07 +0530 Subject: [PATCH 034/174] fix(auth)- Fix Redirect url flow in OidcCallback (#11878) --- datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java index ef5833f607efdb..113aeeb36551f0 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java @@ -130,8 +130,6 @@ public Object perform( CallContext ctx = ctxResult.getFirst(); Result result = (Result) ctxResult.getSecond(); - setContextRedirectUrl(ctx); - // Handle OIDC authentication errors. if (OidcResponseErrorHandler.isError(ctx)) { return OidcResponseErrorHandler.handleError(ctx); @@ -192,6 +190,9 @@ private Pair superPerform( } } + // Set the redirect url from cookie before creating action + setContextRedirectUrl(ctx); + action = this.redirectToOriginallyRequestedUrl(ctx, defaultUrl); } } catch (RuntimeException var20) { From 7dbb3e60cbefe8da96985a2de607eb2ef18514f4 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Nov 2024 13:33:30 -0800 Subject: [PATCH 035/174] chore(ingest): start using explicit exports (#11899) --- metadata-ingestion/scripts/avro_codegen.py | 4 ++-- metadata-ingestion/setup.cfg | 10 +++++++- .../datahub/api/circuit_breaker/__init__.py | 7 ++++++ .../api/circuit_breaker/circuit_breaker.py | 2 +- .../datahub/api/entities/datajob/__init__.py | 3 +++ .../datahub/api/entities/datajob/dataflow.py | 3 +-- .../datahub/api/entities/datajob/datajob.py | 3 +-- .../src/datahub/api/graphql/__init__.py | 2 ++ metadata-ingestion/src/datahub/cli/put_cli.py | 3 ++- .../src/datahub/configuration/__init__.py | 5 ++-- .../src/datahub/configuration/common.py | 2 +- .../src/datahub/configuration/json_loader.py | 2 +- .../datahub/configuration/source_common.py | 8 ++----- .../src/datahub/configuration/yaml.py | 2 +- .../src/datahub/emitter/mce_builder.py | 6 ++++- metadata-ingestion/src/datahub/entrypoints.py | 7 ++---- .../src/datahub/ingestion/api/decorators.py | 5 +++- .../ingestion/extractor/json_schema_util.py | 6 ++--- .../ingestion/extractor/protobuf_util.py | 6 ++--- .../src/datahub/ingestion/graph/client.py | 4 +++- .../datahub_ingestion_run_summary_provider.py | 3 +-- .../ingestion/source/dbt/dbt_common.py | 3 +-- .../ingestion/source/delta_lake/config.py | 3 +-- .../ingestion/source/dynamodb/dynamodb.py | 18 +++++++-------- .../ingestion/source/fivetran/config.py | 3 ++- .../ingestion/source/looker/looker_common.py | 10 ++++---- .../source/looker/looker_dataclasses.py | 10 ++++---- .../source/looker/looker_file_loader.py | 6 ++--- .../ingestion/source/looker/looker_usage.py | 3 ++- .../source/looker/looker_view_id_cache.py | 4 ++-- .../ingestion/source/looker/lookml_config.py | 14 ++++------- .../source/looker/lookml_refinement.py | 2 +- .../ingestion/source/looker/lookml_source.py | 23 ++++++++++--------- .../ingestion/source/looker/view_upstream.py | 2 +- .../src/datahub/ingestion/source/mongodb.py | 14 +++++------ .../ingestion/source/powerbi/config.py | 4 ++-- .../src/datahub/ingestion/source/redash.py | 2 +- .../src/datahub/ingestion/source/sac/sac.py | 7 ++---- .../source/schema_inference/csv_tsv.py | 6 ++--- .../ingestion/source/schema_inference/json.py | 6 ++--- .../source/schema_inference/parquet.py | 6 ++--- .../source/snowflake/snowflake_lineage_v2.py | 8 ++++--- .../source/snowflake/snowflake_v2.py | 6 ++--- .../src/datahub/specific/dataset.py | 4 ++-- .../src/datahub/sql_parsing/_models.py | 2 ++ .../datahub/sql_parsing/schema_resolver.py | 2 +- .../testing/check_sql_parser_result.py | 8 ++----- .../datahub/utilities/urns/corp_group_urn.py | 4 +++- .../datahub/utilities/urns/corpuser_urn.py | 4 +++- .../datahub/utilities/urns/data_flow_urn.py | 4 +++- .../datahub/utilities/urns/data_job_urn.py | 4 +++- .../utilities/urns/data_platform_urn.py | 4 +++- .../urns/data_process_instance_urn.py | 4 +++- .../src/datahub/utilities/urns/dataset_urn.py | 4 +++- .../src/datahub/utilities/urns/domain_urn.py | 4 +++- .../datahub/utilities/urns/notebook_urn.py | 4 +++- .../urns/structured_properties_urn.py | 4 +++- .../src/datahub/utilities/urns/tag_urn.py | 4 +++- .../src/datahub/utilities/urns/urn.py | 4 +++- .../tests/integration/lookml/test_lookml.py | 8 +++---- .../tableau/test_tableau_ingest.py | 3 +-- .../tests/test_helpers/docker_helpers.py | 8 +++---- .../tests/test_helpers/mce_helpers.py | 9 ++++---- .../tests/test_helpers/type_helpers.py | 7 ------ .../tests/unit/glue/test_glue_source.py | 13 +++++------ .../unit/redshift/test_redshift_source.py | 10 ++++---- .../tests/unit/serde/test_serde.py | 11 ++++----- .../unit/sql_parsing/test_sqlglot_utils.py | 6 ++--- .../state/test_stateful_ingestion.py | 3 ++- 69 files changed, 206 insertions(+), 189 deletions(-) diff --git a/metadata-ingestion/scripts/avro_codegen.py b/metadata-ingestion/scripts/avro_codegen.py index e2dd5151439923..e5792da32fb5d7 100644 --- a/metadata-ingestion/scripts/avro_codegen.py +++ b/metadata-ingestion/scripts/avro_codegen.py @@ -769,7 +769,7 @@ def generate( import importlib from typing import TYPE_CHECKING -from datahub._codegen.aspect import _Aspect +from datahub._codegen.aspect import _Aspect as _Aspect from datahub.utilities.docs_build import IS_SPHINX_BUILD from datahub.utilities._custom_package_loader import get_custom_models_package @@ -802,7 +802,7 @@ def generate( from datahub.utilities.docs_build import IS_SPHINX_BUILD from datahub.utilities._custom_package_loader import get_custom_urns_package -from datahub.utilities.urns._urn_base import Urn # noqa: F401 +from datahub.utilities.urns._urn_base import Urn as Urn # noqa: F401 _custom_package_path = get_custom_urns_package() diff --git a/metadata-ingestion/setup.cfg b/metadata-ingestion/setup.cfg index c095420e4e3f30..057779bc87c622 100644 --- a/metadata-ingestion/setup.cfg +++ b/metadata-ingestion/setup.cfg @@ -31,7 +31,7 @@ exclude = __pycache__ per-file-ignores = # imported but unused - __init__.py: F401 + __init__.py: F401, I250 ban-relative-imports = true [mypy] @@ -53,6 +53,14 @@ disallow_untyped_defs = no # try to be a bit more strict in certain areas of the codebase [mypy-datahub.*] ignore_missing_imports = no +implicit_reexport = no +[mypy-datahub.metadata.*] +# TODO: Remove this once all the code has been updated. +implicit_reexport = yes +[mypy-datahub.ingestion.*] +# TODO: Remove this once all the code has been updated. +implicit_reexport = yes + [mypy-datahub_provider.*] ignore_missing_imports = no [mypy-tests.*] diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py b/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py index 27317826264b85..0b04bfa4025a1b 100644 --- a/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py +++ b/metadata-ingestion/src/datahub/api/circuit_breaker/__init__.py @@ -12,3 +12,10 @@ ) requests_logger.setLevel(logging.WARNING) + +__all__ = [ + "AssertionCircuitBreaker", + "AssertionCircuitBreakerConfig", + "OperationCircuitBreaker", + "OperationCircuitBreakerConfig", +] diff --git a/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py b/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py index a3c54046faf681..7c1180536a90fb 100644 --- a/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py +++ b/metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py @@ -6,7 +6,7 @@ from gql.transport.requests import RequestsHTTPTransport from pydantic import Field -from datahub.configuration import ConfigModel +from datahub.configuration.common import ConfigModel logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py b/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py index 6d85a1569cb63d..3a073005968222 100644 --- a/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py +++ b/metadata-ingestion/src/datahub/api/entities/datajob/__init__.py @@ -1,2 +1,5 @@ from datahub.api.entities.datajob.dataflow import DataFlow from datahub.api.entities.datajob.datajob import DataJob + +# TODO: Remove this and start importing directly from the inner files. +__all__ = ["DataFlow", "DataJob"] diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py index f2436d56d5aca1..e169c07445e969 100644 --- a/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py +++ b/metadata-ingestion/src/datahub/api/entities/datajob/dataflow.py @@ -3,7 +3,6 @@ from typing import Callable, Dict, Iterable, List, Optional, Set, cast import datahub.emitter.mce_builder as builder -from datahub.configuration.source_common import ALL_ENV_TYPES from datahub.emitter.generic_emitter import Emitter from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.metadata.schema_classes import ( @@ -114,7 +113,7 @@ def generate_tags_aspect(self) -> List[GlobalTagsClass]: def _get_env(self) -> Optional[str]: env: Optional[str] = None - if self.env and self.env.upper() in ALL_ENV_TYPES: + if self.env and self.env.upper() in builder.ALL_ENV_TYPES: env = self.env.upper() else: logger.debug( diff --git a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py index 0f5d18c20e055b..4958a68caa95fe 100644 --- a/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py +++ b/metadata-ingestion/src/datahub/api/entities/datajob/datajob.py @@ -3,7 +3,6 @@ from typing import Callable, Dict, Iterable, List, Optional, Set import datahub.emitter.mce_builder as builder -from datahub.configuration.source_common import ALL_ENV_TYPES from datahub.emitter.generic_emitter import Emitter from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.metadata.schema_classes import ( @@ -109,7 +108,7 @@ def generate_mcp( self, materialize_iolets: bool = True ) -> Iterable[MetadataChangeProposalWrapper]: env: Optional[str] = None - if self.flow_urn.cluster.upper() in ALL_ENV_TYPES: + if self.flow_urn.cluster.upper() in builder.ALL_ENV_TYPES: env = self.flow_urn.cluster.upper() else: logger.debug( diff --git a/metadata-ingestion/src/datahub/api/graphql/__init__.py b/metadata-ingestion/src/datahub/api/graphql/__init__.py index e8c8d22bbb93df..d818b19092fcbe 100644 --- a/metadata-ingestion/src/datahub/api/graphql/__init__.py +++ b/metadata-ingestion/src/datahub/api/graphql/__init__.py @@ -1,2 +1,4 @@ from datahub.api.graphql.assertion import Assertion from datahub.api.graphql.operation import Operation + +__all__ = ["Assertion", "Operation"] diff --git a/metadata-ingestion/src/datahub/cli/put_cli.py b/metadata-ingestion/src/datahub/cli/put_cli.py index 989b1a6d02fd01..0a40a9f4ccf92d 100644 --- a/metadata-ingestion/src/datahub/cli/put_cli.py +++ b/metadata-ingestion/src/datahub/cli/put_cli.py @@ -6,11 +6,12 @@ from datahub.cli.cli_utils import post_entity from datahub.configuration.config_loader import load_config_file -from datahub.emitter.mcp import MetadataChangeProposalWrapper, SystemMetadataClass +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.graph.client import get_default_graph from datahub.metadata.schema_classes import ( DataPlatformInfoClass as DataPlatformInfo, PlatformTypeClass, + SystemMetadataClass, ) from datahub.telemetry import telemetry from datahub.upgrade import upgrade diff --git a/metadata-ingestion/src/datahub/configuration/__init__.py b/metadata-ingestion/src/datahub/configuration/__init__.py index 008d788072d0a5..21979829a4453d 100644 --- a/metadata-ingestion/src/datahub/configuration/__init__.py +++ b/metadata-ingestion/src/datahub/configuration/__init__.py @@ -1,5 +1,4 @@ from datahub.configuration.common import ( - ConfigModel, - ConfigurationMechanism, - DynamicTypedConfig, + ConfigModel as ConfigModel, + DynamicTypedConfig as DynamicTypedConfig, ) diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 0ce7127b440534..4fdf564162410c 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -21,7 +21,7 @@ from pydantic.fields import Field from typing_extensions import Protocol -from datahub.configuration._config_enum import ConfigEnum +from datahub.configuration._config_enum import ConfigEnum as ConfigEnum # noqa: I250 from datahub.configuration.pydantic_migration_helpers import PYDANTIC_VERSION_2 from datahub.utilities.dedup_list import deduplicate_list diff --git a/metadata-ingestion/src/datahub/configuration/json_loader.py b/metadata-ingestion/src/datahub/configuration/json_loader.py index 35667eb5951fc7..6ecb741be528d1 100644 --- a/metadata-ingestion/src/datahub/configuration/json_loader.py +++ b/metadata-ingestion/src/datahub/configuration/json_loader.py @@ -1,7 +1,7 @@ import json from typing import IO -from datahub.configuration import ConfigurationMechanism +from datahub.configuration.common import ConfigurationMechanism class JsonConfigurationMechanism(ConfigurationMechanism): diff --git a/metadata-ingestion/src/datahub/configuration/source_common.py b/metadata-ingestion/src/datahub/configuration/source_common.py index ad12447532335f..44c737f1bd13d4 100644 --- a/metadata-ingestion/src/datahub/configuration/source_common.py +++ b/metadata-ingestion/src/datahub/configuration/source_common.py @@ -1,14 +1,10 @@ -from typing import Dict, Optional, Set +from typing import Dict, Optional from pydantic import validator from pydantic.fields import Field from datahub.configuration.common import ConfigModel -from datahub.emitter.enum_helpers import get_enum_options -from datahub.metadata.schema_classes import FabricTypeClass - -DEFAULT_ENV = FabricTypeClass.PROD -ALL_ENV_TYPES: Set[str] = set(get_enum_options(FabricTypeClass)) +from datahub.emitter.mce_builder import ALL_ENV_TYPES, DEFAULT_ENV class PlatformInstanceConfigMixin(ConfigModel): diff --git a/metadata-ingestion/src/datahub/configuration/yaml.py b/metadata-ingestion/src/datahub/configuration/yaml.py index 1f1172836f7448..c069845e1de119 100644 --- a/metadata-ingestion/src/datahub/configuration/yaml.py +++ b/metadata-ingestion/src/datahub/configuration/yaml.py @@ -2,7 +2,7 @@ import yaml -from datahub.configuration import ConfigurationMechanism +from datahub.configuration.common import ConfigurationMechanism class YamlConfigurationMechanism(ConfigurationMechanism): diff --git a/metadata-ingestion/src/datahub/emitter/mce_builder.py b/metadata-ingestion/src/datahub/emitter/mce_builder.py index 63b03db7f5b604..69946c575908b5 100644 --- a/metadata-ingestion/src/datahub/emitter/mce_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mce_builder.py @@ -13,6 +13,7 @@ Any, List, Optional, + Set, Tuple, Type, TypeVar, @@ -24,7 +25,6 @@ import typing_inspect from avrogen.dict_wrapper import DictWrapper -from datahub.configuration.source_common import DEFAULT_ENV from datahub.emitter.enum_helpers import get_enum_options from datahub.metadata.schema_classes import ( AssertionKeyClass, @@ -35,6 +35,7 @@ DatasetKeyClass, DatasetLineageTypeClass, DatasetSnapshotClass, + FabricTypeClass, GlobalTagsClass, GlossaryTermAssociationClass, GlossaryTermsClass as GlossaryTerms, @@ -56,6 +57,9 @@ logger = logging.getLogger(__name__) Aspect = TypeVar("Aspect", bound=AspectAbstract) +DEFAULT_ENV = FabricTypeClass.PROD +ALL_ENV_TYPES: Set[str] = set(get_enum_options(FabricTypeClass)) + DEFAULT_FLOW_CLUSTER = "prod" UNKNOWN_USER = "urn:li:corpuser:unknown" DATASET_URN_TO_LOWER: bool = ( diff --git a/metadata-ingestion/src/datahub/entrypoints.py b/metadata-ingestion/src/datahub/entrypoints.py index d088380d5d38c4..85968f050a3716 100644 --- a/metadata-ingestion/src/datahub/entrypoints.py +++ b/metadata-ingestion/src/datahub/entrypoints.py @@ -13,13 +13,10 @@ generate_access_token, make_shim_command, ) -from datahub.cli.config_utils import ( - DATAHUB_CONFIG_PATH, - get_boolean_env_variable, - write_gms_config, -) +from datahub.cli.config_utils import DATAHUB_CONFIG_PATH, write_gms_config from datahub.cli.delete_cli import delete from datahub.cli.docker_cli import docker +from datahub.cli.env_utils import get_boolean_env_variable from datahub.cli.exists_cli import exists from datahub.cli.get_cli import get from datahub.cli.ingest_cli import ingest diff --git a/metadata-ingestion/src/datahub/ingestion/api/decorators.py b/metadata-ingestion/src/datahub/ingestion/api/decorators.py index b390ffb9dd0362..d32c0b85ceef4c 100644 --- a/metadata-ingestion/src/datahub/ingestion/api/decorators.py +++ b/metadata-ingestion/src/datahub/ingestion/api/decorators.py @@ -3,7 +3,10 @@ from typing import Callable, Dict, Optional, Type from datahub.ingestion.api.common import PipelineContext -from datahub.ingestion.api.source import Source, SourceCapability +from datahub.ingestion.api.source import ( # noqa: I250 + Source, + SourceCapability as SourceCapability, +) def config_class(config_cls: Type) -> Callable[[Type], Type]: diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py index bcf077154343c8..88d1fcc52e2196 100644 --- a/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py +++ b/metadata-ingestion/src/datahub/ingestion/extractor/json_schema_util.py @@ -23,7 +23,7 @@ RecordTypeClass, SchemaFieldClass as SchemaField, SchemaFieldDataTypeClass, - SchemaMetadataClass as SchemaMetadata, + SchemaMetadataClass, StringTypeClass, UnionTypeClass, ) @@ -665,13 +665,13 @@ def get_schema_metadata( name: str, json_schema: Dict[Any, Any], raw_schema_string: Optional[str] = None, -) -> SchemaMetadata: +) -> SchemaMetadataClass: json_schema_as_string = raw_schema_string or json.dumps(json_schema) md5_hash: str = md5(json_schema_as_string.encode()).hexdigest() schema_fields = list(JsonSchemaTranslator.get_fields_from_schema(json_schema)) - schema_metadata = SchemaMetadata( + schema_metadata = SchemaMetadataClass( schemaName=name, platform=f"urn:li:dataPlatform:{platform}", version=0, diff --git a/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py b/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py index f62bb184252d98..e947aff384871d 100644 --- a/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py +++ b/metadata-ingestion/src/datahub/ingestion/extractor/protobuf_util.py @@ -32,7 +32,7 @@ OneofDescriptor, ) -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( +from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, BytesTypeClass, @@ -41,8 +41,8 @@ MapTypeClass, NumberTypeClass, RecordTypeClass, - SchemaField, - SchemaFieldDataType, + SchemaFieldClass as SchemaField, + SchemaFieldDataTypeClass as SchemaFieldDataType, StringTypeClass, UnionTypeClass, ) diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index c90ac93eee2cc2..759aebcfd46b0a 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -33,7 +33,9 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.rest_emitter import DatahubRestEmitter from datahub.emitter.serialization_helper import post_json_transform -from datahub.ingestion.graph.config import DatahubClientConfig +from datahub.ingestion.graph.config import ( # noqa: I250; TODO: Remove this alias + DatahubClientConfig as DatahubClientConfig, +) from datahub.ingestion.graph.connections import ( connections_gql, get_id_from_connection_urn, diff --git a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py index 33bfb63feb3fd7..5961a553a14943 100644 --- a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +++ b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py @@ -11,9 +11,8 @@ redact_raw_config, ) from datahub.emitter.aspect import JSON_CONTENT_TYPE -from datahub.emitter.mce_builder import datahub_guid +from datahub.emitter.mce_builder import datahub_guid, make_data_platform_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import make_data_platform_urn from datahub.ingestion.api.common import PipelineContext, RecordEnvelope from datahub.ingestion.api.pipeline_run_listener import PipelineRunListener from datahub.ingestion.api.sink import NoopWriteCallback, Sink diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index b5d0ed42e651ea..4598ae388b827d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -117,9 +117,8 @@ ViewPropertiesClass, ) from datahub.metadata.urns import DatasetUrn -from datahub.sql_parsing.schema_resolver import SchemaResolver +from datahub.sql_parsing.schema_resolver import SchemaInfo, SchemaResolver from datahub.sql_parsing.sqlglot_lineage import ( - SchemaInfo, SqlParsingDebugInfo, SqlParsingResult, infer_output_schema, diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py index 81a54d1327d05a..d2b4a576953daf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/config.py @@ -6,9 +6,8 @@ from pydantic import Field from typing_extensions import Literal -from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.common import AllowDenyPattern, ConfigModel from datahub.configuration.source_common import ( - ConfigModel, EnvConfigMixin, PlatformInstanceConfigMixin, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py index acda656526ef53..4f1de6fb06c695 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py @@ -52,24 +52,22 @@ from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionSourceBase, ) -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( +from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, BytesTypeClass, + DataPlatformInstanceClass, + DatasetPropertiesClass, NullTypeClass, NumberTypeClass, RecordTypeClass, - SchemaField, - SchemaFieldDataType, + SchemaFieldClass as SchemaField, + SchemaFieldDataTypeClass as SchemaFieldDataType, SchemalessClass, - SchemaMetadata, + SchemaMetadataClass, StringTypeClass, UnionTypeClass, ) -from datahub.metadata.schema_classes import ( - DataPlatformInstanceClass, - DatasetPropertiesClass, -) from datahub.utilities.registries.domain_registry import DomainRegistry MAX_ITEMS_TO_RETRIEVE = 100 @@ -448,7 +446,7 @@ def construct_schema_metadata( dataset_properties: DatasetPropertiesClass, schema: Dict[Tuple[str, ...], SchemaDescription], primary_key_dict: Dict[str, str], - ) -> SchemaMetadata: + ) -> SchemaMetadataClass: """ " To construct the schema metadata, it will first sort the schema by the occurrence of attribute names in descending order and truncate the schema by MAX_SCHEMA_SIZE, and then start to construct the @@ -502,7 +500,7 @@ def construct_schema_metadata( canonical_schema.append(field) # create schema metadata object for table - schema_metadata = SchemaMetadata( + schema_metadata = SchemaMetadataClass( schemaName=table_name, platform=f"urn:li:dataPlatform:{self.platform}", version=0, diff --git a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py index e40e284d6e0a42..86826ae7bedc09 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/fivetran/config.py @@ -12,8 +12,9 @@ ConfigModel, ConfigurationWarning, ) -from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin +from datahub.configuration.source_common import DatasetSourceConfigMixin from datahub.configuration.validate_field_rename import pydantic_renamed_field +from datahub.emitter.mce_builder import DEFAULT_ENV from datahub.ingestion.api.report import Report from datahub.ingestion.source.bigquery_v2.bigquery_config import ( BigQueryConnectionConfig, diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index 3d1683100474e8..3e2872a4b5caa1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -48,7 +48,7 @@ from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader from datahub.ingestion.source.looker.looker_lib_wrapper import LookerAPI from datahub.ingestion.source.looker.lookml_config import ( - _BASE_PROJECT_NAME, + BASE_PROJECT_NAME, LookMLSourceReport, ) from datahub.ingestion.source.looker.str_functions import remove_suffix @@ -370,7 +370,7 @@ def _form_field_name( assert view_name # for lint false positive project_include: ProjectInclude = ProjectInclude( - project=view_project_map.get(view_name, _BASE_PROJECT_NAME), + project=view_project_map.get(view_name, BASE_PROJECT_NAME), include=view_name, ) @@ -385,7 +385,7 @@ def _form_field_name( view_urn = LookerViewId( project_name=( project_include.project - if project_include.project != _BASE_PROJECT_NAME + if project_include.project != BASE_PROJECT_NAME else explore_project_name ), model_name=model_name, @@ -1113,7 +1113,7 @@ def from_api( # noqa: C901 fields=view_fields, upstream_views=list( ProjectInclude( - project=view_project_map.get(view_name, _BASE_PROJECT_NAME), + project=view_project_map.get(view_name, BASE_PROJECT_NAME), include=view_name, ) for view_name in views @@ -1239,7 +1239,7 @@ def _to_metadata_events( # noqa: C901 view_urn = LookerViewId( project_name=( view_ref.project - if view_ref.project != _BASE_PROJECT_NAME + if view_ref.project != BASE_PROJECT_NAME else self.project_name ), model_name=self.model_name, diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py index 7e23079156b625..327c9ebf99bd20 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_dataclasses.py @@ -9,8 +9,8 @@ load_and_preprocess_file, ) from datahub.ingestion.source.looker.lookml_config import ( - _BASE_PROJECT_NAME, - _EXPLORE_FILE_EXTENSION, + BASE_PROJECT_NAME, + EXPLORE_FILE_EXTENSION, LookMLSourceConfig, LookMLSourceReport, ) @@ -69,7 +69,7 @@ def from_looker_dict( explore_files = [ x.include for x in resolved_includes - if x.include.endswith(_EXPLORE_FILE_EXTENSION) + if x.include.endswith(EXPLORE_FILE_EXTENSION) ] for included_file in explore_files: try: @@ -152,9 +152,9 @@ def resolve_includes( # As such, we try to handle it but are as defensive as possible. non_base_project_name = project_name - if project_name == _BASE_PROJECT_NAME and root_project_name is not None: + if project_name == BASE_PROJECT_NAME and root_project_name is not None: non_base_project_name = root_project_name - if non_base_project_name != _BASE_PROJECT_NAME and inc.startswith( + if non_base_project_name != BASE_PROJECT_NAME and inc.startswith( f"/{non_base_project_name}/" ): # This might be a local include. Let's make sure that '/{project_name}' doesn't diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py index f894c96debc54a..9fac0b52fde0dd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_file_loader.py @@ -9,8 +9,8 @@ load_and_preprocess_file, ) from datahub.ingestion.source.looker.lookml_config import ( - _EXPLORE_FILE_EXTENSION, - _VIEW_FILE_EXTENSION, + EXPLORE_FILE_EXTENSION, + VIEW_FILE_EXTENSION, LookMLSourceConfig, LookMLSourceReport, ) @@ -42,7 +42,7 @@ def _load_viewfile( ) -> Optional[LookerViewFile]: # always fully resolve paths to simplify de-dup path = str(pathlib.Path(path).resolve()) - allowed_extensions = [_VIEW_FILE_EXTENSION, _EXPLORE_FILE_EXTENSION] + allowed_extensions = [VIEW_FILE_EXTENSION, EXPLORE_FILE_EXTENSION] matched_any_extension = [ match for match in [path.endswith(x) for x in allowed_extensions] if match ] diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py index 6a623e1e97b5dc..ef7d64e4f42d43 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py @@ -14,7 +14,7 @@ from looker_sdk.sdk.api40.models import Dashboard, LookWithQuery -from datahub.emitter.mce_builder import Aspect, AspectAbstract +from datahub.emitter.mce_builder import Aspect from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.source.looker import looker_common from datahub.ingestion.source.looker.looker_common import ( @@ -40,6 +40,7 @@ DashboardUsageStatisticsClass, DashboardUserUsageCountsClass, TimeWindowSizeClass, + _Aspect as AspectAbstract, ) logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py index aa45bb72d1f462..562c7863b31343 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_view_id_cache.py @@ -6,7 +6,7 @@ from datahub.ingestion.source.looker.looker_dataclasses import LookerModel from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader from datahub.ingestion.source.looker.lookml_config import ( - _BASE_PROJECT_NAME, + BASE_PROJECT_NAME, NAME, LookMLSourceReport, ) @@ -103,7 +103,7 @@ def get_looker_view_id( current_project_name: str = ( include.project - if include.project != _BASE_PROJECT_NAME + if include.project != BASE_PROJECT_NAME else self.project_name ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py index da837da1613864..7ffb895349ed29 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_config.py @@ -33,17 +33,11 @@ NAME: str = "name" -_BASE_PROJECT_NAME = "__BASE" +BASE_PROJECT_NAME = "__BASE" -_EXPLORE_FILE_EXTENSION = ".explore.lkml" - -_VIEW_FILE_EXTENSION = ".view.lkml" - -_MODEL_FILE_EXTENSION = ".model.lkml" - -VIEW_LANGUAGE_LOOKML: str = "lookml" - -VIEW_LANGUAGE_SQL: str = "sql" +EXPLORE_FILE_EXTENSION = ".explore.lkml" +VIEW_FILE_EXTENSION = ".view.lkml" +MODEL_FILE_EXTENSION = ".model.lkml" DERIVED_VIEW_SUFFIX = r".sql_table_name" diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py index 892ed79754a1c2..6933d9d69394bc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_refinement.py @@ -5,7 +5,7 @@ from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition from datahub.ingestion.source.looker.looker_dataclasses import LookerModel -from datahub.ingestion.source.looker.looker_view_id_cache import LookerViewFileLoader +from datahub.ingestion.source.looker.looker_file_loader import LookerViewFileLoader from datahub.ingestion.source.looker.lookml_config import ( NAME, LookMLSourceConfig, diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index d258570ec384f7..3c83b8728aa6f7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -57,10 +57,8 @@ LookerViewContext, ) from datahub.ingestion.source.looker.lookml_config import ( - _BASE_PROJECT_NAME, - _MODEL_FILE_EXTENSION, - VIEW_LANGUAGE_LOOKML, - VIEW_LANGUAGE_SQL, + BASE_PROJECT_NAME, + MODEL_FILE_EXTENSION, LookerConnectionDefinition, LookMLSourceConfig, LookMLSourceReport, @@ -98,6 +96,9 @@ ) from datahub.sql_parsing.sqlglot_lineage import ColumnRef +VIEW_LANGUAGE_LOOKML: str = "lookml" +VIEW_LANGUAGE_SQL: str = "sql" + logger = logging.getLogger(__name__) @@ -319,7 +320,7 @@ def _load_model(self, path: str) -> LookerModel: looker_model = LookerModel.from_looker_dict( parsed, - _BASE_PROJECT_NAME, + BASE_PROJECT_NAME, self.source_config.project_name, self.base_projects_folder, path, @@ -544,7 +545,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.source_config.base_folder = checkout_dir.resolve() self.base_projects_folder[ - _BASE_PROJECT_NAME + BASE_PROJECT_NAME ] = self.source_config.base_folder visited_projects: Set[str] = set() @@ -576,7 +577,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self.base_projects_folder[project] = p_ref self._recursively_check_manifests( - tmp_dir, _BASE_PROJECT_NAME, visited_projects + tmp_dir, BASE_PROJECT_NAME, visited_projects ) yield from self.get_internal_workunits() @@ -607,7 +608,7 @@ def _recursively_check_manifests( return # Special case handling if the root project has a name in the manifest file. - if project_name == _BASE_PROJECT_NAME and manifest.project_name: + if project_name == BASE_PROJECT_NAME and manifest.project_name: if ( self.source_config.project_name is not None and manifest.project_name != self.source_config.project_name @@ -696,7 +697,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 # The ** means "this directory and all subdirectories", and hence should # include all the files we want. model_files = sorted( - self.source_config.base_folder.glob(f"**/*{_MODEL_FILE_EXTENSION}") + self.source_config.base_folder.glob(f"**/*{MODEL_FILE_EXTENSION}") ) model_suffix_len = len(".model") @@ -832,7 +833,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 current_project_name: str = ( include.project - if include.project != _BASE_PROJECT_NAME + if include.project != BASE_PROJECT_NAME else project_name ) @@ -841,7 +842,7 @@ def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]: # noqa: C901 base_folder_path: str = str( self.base_projects_folder.get( current_project_name, - self.base_projects_folder[_BASE_PROJECT_NAME], + self.base_projects_folder[BASE_PROJECT_NAME], ) ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py index 057dbca4281849..632d0caf712323 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py @@ -12,6 +12,7 @@ ViewField, ViewFieldType, ) +from datahub.ingestion.source.looker.looker_config import LookerConnectionDefinition from datahub.ingestion.source.looker.looker_view_id_cache import LookerViewIdCache from datahub.ingestion.source.looker.lookml_concept_context import ( LookerFieldContext, @@ -20,7 +21,6 @@ from datahub.ingestion.source.looker.lookml_config import ( DERIVED_VIEW_SUFFIX, NAME, - LookerConnectionDefinition, LookMLSourceConfig, LookMLSourceReport, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py index c87b025f13b55d..bbc4897d227bac 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py @@ -50,25 +50,23 @@ from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionSourceBase, ) -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( +from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, BytesTypeClass, + DataPlatformInstanceClass, + DatasetPropertiesClass, NullTypeClass, NumberTypeClass, RecordTypeClass, - SchemaField, - SchemaFieldDataType, + SchemaFieldClass as SchemaField, + SchemaFieldDataTypeClass as SchemaFieldDataType, SchemalessClass, - SchemaMetadata, + SchemaMetadataClass as SchemaMetadata, StringTypeClass, TimeTypeClass, UnionTypeClass, ) -from datahub.metadata.schema_classes import ( - DataPlatformInstanceClass, - DatasetPropertiesClass, -) from datahub.metadata.urns import DatasetUrn logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index 7c8487727c9eee..91fa2e96be2cce 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -9,7 +9,7 @@ import datahub.emitter.mce_builder as builder from datahub.configuration.common import AllowDenyPattern, ConfigModel -from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin +from datahub.configuration.source_common import DatasetSourceConfigMixin from datahub.configuration.validate_field_deprecation import pydantic_field_deprecated from datahub.ingestion.source.common.subtypes import BIAssetSubTypes from datahub.ingestion.source.state.stale_entity_removal_handler import ( @@ -240,7 +240,7 @@ class PlatformDetail(ConfigModel): "recipe of other datahub sources.", ) env: str = pydantic.Field( - default=DEFAULT_ENV, + default=builder.DEFAULT_ENV, description="The environment that all assets produced by DataHub platform ingestion source belong to", ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py index 5fd63e7f93f92a..581e32d29dceaf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redash.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py @@ -41,7 +41,7 @@ ) from datahub.utilities.lossy_collections import LossyDict, LossyList from datahub.utilities.perf_timer import PerfTimer -from datahub.utilities.sql_parser import SQLParser +from datahub.utilities.sql_parser_base import SQLParser from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py index de0904107b9bbe..66962b5d96d389 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py @@ -13,12 +13,9 @@ from urllib3.util.retry import Retry from datahub.configuration.common import AllowDenyPattern -from datahub.configuration.source_common import ( - DEFAULT_ENV, - DatasetSourceConfigMixin, - EnvConfigMixin, -) +from datahub.configuration.source_common import DatasetSourceConfigMixin, EnvConfigMixin from datahub.emitter.mce_builder import ( + DEFAULT_ENV, dataset_urn_to_key, make_dashboard_urn, make_data_platform_urn, diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py index 54f7dfb5b903c7..ab7b887cba1d80 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py +++ b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/csv_tsv.py @@ -3,15 +3,15 @@ from tableschema import Table from datahub.ingestion.source.schema_inference.base import SchemaInferenceBase -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( +from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, DateTypeClass, NullTypeClass, NumberTypeClass, RecordTypeClass, - SchemaField, - SchemaFieldDataType, + SchemaFieldClass as SchemaField, + SchemaFieldDataTypeClass as SchemaFieldDataType, StringTypeClass, TimeTypeClass, UnionTypeClass, diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py index 1f2c73a2522d04..1659aaf6fa2020 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py +++ b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/json.py @@ -7,14 +7,14 @@ from datahub.ingestion.source.schema_inference.base import SchemaInferenceBase from datahub.ingestion.source.schema_inference.object import construct_schema -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( +from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, NullTypeClass, NumberTypeClass, RecordTypeClass, - SchemaField, - SchemaFieldDataType, + SchemaFieldClass as SchemaField, + SchemaFieldDataTypeClass as SchemaFieldDataType, StringTypeClass, UnionTypeClass, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py index 1f3f2e0a1e8a83..efc605e0df8cab 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py +++ b/metadata-ingestion/src/datahub/ingestion/source/schema_inference/parquet.py @@ -4,7 +4,7 @@ import pyarrow.parquet from datahub.ingestion.source.schema_inference.base import SchemaInferenceBase -from datahub.metadata.com.linkedin.pegasus2avro.schema import ( +from datahub.metadata.schema_classes import ( ArrayTypeClass, BooleanTypeClass, BytesTypeClass, @@ -12,8 +12,8 @@ NullTypeClass, NumberTypeClass, RecordTypeClass, - SchemaField, - SchemaFieldDataType, + SchemaFieldClass as SchemaField, + SchemaFieldDataTypeClass as SchemaFieldDataType, StringTypeClass, TimeTypeClass, UnionTypeClass, diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index 6f9c9259b27844..ac47abf4874499 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -31,14 +31,16 @@ ) from datahub.metadata.schema_classes import DatasetLineageTypeClass, UpstreamClass from datahub.sql_parsing.sql_parsing_aggregator import ( - ColumnLineageInfo, - ColumnRef, KnownLineageMapping, KnownQueryLineageInfo, SqlParsingAggregator, UrnStr, ) -from datahub.sql_parsing.sqlglot_lineage import DownstreamColumnRef +from datahub.sql_parsing.sqlglot_lineage import ( + ColumnLineageInfo, + ColumnRef, + DownstreamColumnRef, +) from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.time import ts_millis_to_datetime diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index dd7f73268fdc4f..538841018067e2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -48,11 +48,9 @@ SnowflakeQueriesExtractor, SnowflakeQueriesExtractorConfig, ) +from datahub.ingestion.source.snowflake.snowflake_query import SnowflakeQuery from datahub.ingestion.source.snowflake.snowflake_report import SnowflakeV2Report -from datahub.ingestion.source.snowflake.snowflake_schema import ( - SnowflakeDataDictionary, - SnowflakeQuery, -) +from datahub.ingestion.source.snowflake.snowflake_schema import SnowflakeDataDictionary from datahub.ingestion.source.snowflake.snowflake_schema_gen import ( SnowflakeSchemaGenerator, ) diff --git a/metadata-ingestion/src/datahub/specific/dataset.py b/metadata-ingestion/src/datahub/specific/dataset.py index 9dd2616078f08d..b171dc4cc2939f 100644 --- a/metadata-ingestion/src/datahub/specific/dataset.py +++ b/metadata-ingestion/src/datahub/specific/dataset.py @@ -13,7 +13,7 @@ KafkaAuditHeaderClass, OwnerClass as Owner, OwnershipTypeClass, - SchemaMetadataClass as SchemaMetadata, + SchemaMetadataClass, SystemMetadataClass, TagAssociationClass as Tag, UpstreamClass as Upstream, @@ -40,7 +40,7 @@ def __init__( self.aspect_name = ( EditableSchemaMetadata.ASPECT_NAME if editable - else SchemaMetadata.ASPECT_NAME + else SchemaMetadataClass.ASPECT_NAME ) self.aspect_field = "editableSchemaFieldInfo" if editable else "schemaFieldInfo" diff --git a/metadata-ingestion/src/datahub/sql_parsing/_models.py b/metadata-ingestion/src/datahub/sql_parsing/_models.py index d92d178b81cf4b..d586e7d6d9045b 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/_models.py +++ b/metadata-ingestion/src/datahub/sql_parsing/_models.py @@ -42,6 +42,8 @@ def __lt__(self, other: "_FrozenModel") -> bool: class _TableName(_FrozenModel): + # TODO: Move this into the schema_resolver.py file. + database: Optional[str] = None db_schema: Optional[str] = None table: str diff --git a/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py b/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py index e7b0527d30d978..e3f2fbc786b437 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py +++ b/metadata-ingestion/src/datahub/sql_parsing/schema_resolver.py @@ -13,7 +13,7 @@ from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier from datahub.metadata.schema_classes import SchemaFieldClass, SchemaMetadataClass from datahub.metadata.urns import DataPlatformUrn -from datahub.sql_parsing._models import _TableName +from datahub.sql_parsing._models import _TableName as _TableName # noqa: I250 from datahub.sql_parsing.sql_parsing_common import PLATFORMS_WITH_CASE_SENSITIVE_TABLES from datahub.utilities.file_backed_collections import ConnectionWrapper, FileBackedDict from datahub.utilities.urns.field_paths import get_simple_field_path_from_v2_field_path diff --git a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py index 72b5f6c5e26e4b..13be45ec1be28d 100644 --- a/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py +++ b/metadata-ingestion/src/datahub/testing/check_sql_parser_result.py @@ -6,12 +6,8 @@ import deepdiff from datahub.ingestion.source.bigquery_v2.bigquery_audit import BigqueryTableIdentifier -from datahub.sql_parsing.schema_resolver import SchemaResolver -from datahub.sql_parsing.sqlglot_lineage import ( - SchemaInfo, - SqlParsingResult, - sqlglot_lineage, -) +from datahub.sql_parsing.schema_resolver import SchemaInfo, SchemaResolver +from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult, sqlglot_lineage logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py b/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py index 37c10769259459..577f90215a6353 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/corp_group_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import CorpGroupUrn # noqa: F401 +from datahub.metadata.urns import CorpGroupUrn + +__all__ = ["CorpGroupUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py b/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py index 5f9ecf65951b95..8acb86be00f6c8 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/corpuser_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import CorpUserUrn as CorpuserUrn # noqa: F401 +from datahub.metadata.urns import CorpUserUrn as CorpuserUrn + +__all__ = ["CorpuserUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py index 5b2b45927c339e..3508ae5c4a3490 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/data_flow_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import DataFlowUrn # noqa: F401 +from datahub.metadata.urns import DataFlowUrn + +__all__ = ["DataFlowUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py index 53e3419ee7ecb2..d003b6c6ad7a88 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/data_job_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import DataJobUrn # noqa: F401 +from datahub.metadata.urns import DataJobUrn + +__all__ = ["DataJobUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py index 9d37e38f256e7f..51e013e715d4fd 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/data_platform_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import DataPlatformUrn # noqa: F401 +from datahub.metadata.urns import DataPlatformUrn + +__all__ = ["DataPlatformUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py b/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py index df6ba797d069c1..22e6b36c5f7aec 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/data_process_instance_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import DataProcessInstanceUrn # noqa: F401 +from datahub.metadata.urns import DataProcessInstanceUrn + +__all__ = ["DataProcessInstanceUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py index 6078ffefc03d85..1652e170599958 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/dataset_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import DatasetUrn # noqa: F401 +from datahub.metadata.urns import DatasetUrn + +__all__ = ["DatasetUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py b/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py index 442a6b27729bba..242a3d8228320d 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/domain_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import DomainUrn # noqa: F401 +from datahub.metadata.urns import DomainUrn + +__all__ = ["DomainUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py b/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py index 60a4f5396aa468..f9b861d7f08524 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/notebook_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import NotebookUrn # noqa: F401 +from datahub.metadata.urns import NotebookUrn + +__all__ = ["NotebookUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py index 5bd36a0656d99e..6774978c7a76d9 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py @@ -1,4 +1,6 @@ -from datahub.metadata.urns import StructuredPropertyUrn # noqa: F401 +from datahub.metadata.urns import StructuredPropertyUrn + +__all__ = ["StructuredPropertyUrn", "make_structured_property_urn"] def make_structured_property_urn(structured_property_id: str) -> str: diff --git a/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py b/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py index 0ac632ee40a015..f66d56a745a961 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/tag_urn.py @@ -1 +1,3 @@ -from datahub.metadata.urns import TagUrn # noqa: F401 +from datahub.metadata.urns import TagUrn + +__all__ = ["TagUrn"] diff --git a/metadata-ingestion/src/datahub/utilities/urns/urn.py b/metadata-ingestion/src/datahub/utilities/urns/urn.py index 2e5cebfd0e8f55..2ded2d4d9b32c0 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/urn.py @@ -1,4 +1,6 @@ -from datahub.metadata.urns import Urn # noqa: F401 +from datahub.metadata.urns import Urn + +__all__ = ["Urn", "guess_entity_type"] def guess_entity_type(urn: str) -> str: diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 94b3b103d0548c..a4cfbd5eadb7f3 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -12,16 +12,14 @@ from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.file import read_metadata_file +from datahub.ingestion.source.looker.looker_dataclasses import LookerModel from datahub.ingestion.source.looker.looker_template_language import ( SpecialVariable, load_and_preprocess_file, resolve_liquid_variable, ) -from datahub.ingestion.source.looker.lookml_source import ( - LookerModel, - LookerRefinementResolver, - LookMLSourceConfig, -) +from datahub.ingestion.source.looker.lookml_config import LookMLSourceConfig +from datahub.ingestion.source.looker.lookml_refinement import LookerRefinementResolver from datahub.metadata.schema_classes import ( DatasetSnapshotClass, MetadataChangeEventClass, diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index edfc41616e44be..62f8f6a654b588 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -19,8 +19,7 @@ ) from tableauserverclient.models.reference_item import ResourceReference -from datahub.configuration.source_common import DEFAULT_ENV -from datahub.emitter.mce_builder import make_schema_field_urn +from datahub.emitter.mce_builder import DEFAULT_ENV, make_schema_field_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.run.pipeline import Pipeline, PipelineContext from datahub.ingestion.source.tableau.tableau import ( diff --git a/metadata-ingestion/tests/test_helpers/docker_helpers.py b/metadata-ingestion/tests/test_helpers/docker_helpers.py index 20aec975787e4e..d0e943bbe63daf 100644 --- a/metadata-ingestion/tests/test_helpers/docker_helpers.py +++ b/metadata-ingestion/tests/test_helpers/docker_helpers.py @@ -4,10 +4,10 @@ import pytest -from datahub.testing.docker_utils import ( # noqa: F401 - docker_compose_runner, - is_responsive, - wait_for_port, +from datahub.testing.docker_utils import ( # noqa: F401,I250 + docker_compose_runner as docker_compose_runner, + is_responsive as is_responsive, + wait_for_port as wait_for_port, ) logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/tests/test_helpers/mce_helpers.py b/metadata-ingestion/tests/test_helpers/mce_helpers.py index 3b59481d8cb022..f4c629df7dba4e 100644 --- a/metadata-ingestion/tests/test_helpers/mce_helpers.py +++ b/metadata-ingestion/tests/test_helpers/mce_helpers.py @@ -17,15 +17,16 @@ Union, ) +import pytest + from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.sink.file import write_metadata_file from datahub.metadata.schema_classes import MetadataChangeEventClass +from datahub.metadata.urns import Urn from datahub.testing.compare_metadata_json import ( assert_metadata_files_equal, load_json_file, ) -from datahub.utilities.urns.urn import Urn -from tests.test_helpers.type_helpers import PytestConfig logger = logging.getLogger(__name__) @@ -77,7 +78,7 @@ def clean_nones(value): def check_golden_file( - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, output_path: Union[str, os.PathLike], golden_path: Union[str, os.PathLike], ignore_paths: Sequence[str] = (), @@ -98,7 +99,7 @@ def check_golden_file( def check_goldens_stream( - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, outputs: List, golden_path: Union[str, os.PathLike], ignore_paths: Sequence[str] = (), diff --git a/metadata-ingestion/tests/test_helpers/type_helpers.py b/metadata-ingestion/tests/test_helpers/type_helpers.py index 154960bbf7fc42..3a2215ed81ca99 100644 --- a/metadata-ingestion/tests/test_helpers/type_helpers.py +++ b/metadata-ingestion/tests/test_helpers/type_helpers.py @@ -1,12 +1,5 @@ from typing import Optional, TypeVar -# The current PytestConfig solution is somewhat ugly and not ideal. -# However, it is currently the best solution available, as the type itself is not -# exported: https://docs.pytest.org/en/stable/reference.html#config. -# As pytest's type support improves, this will likely change. -# TODO: revisit pytestconfig as https://github.com/pytest-dev/pytest/issues/7469 progresses. -from _pytest.config import Config as PytestConfig # noqa: F401 - _T = TypeVar("_T") diff --git a/metadata-ingestion/tests/unit/glue/test_glue_source.py b/metadata-ingestion/tests/unit/glue/test_glue_source.py index 4df0c6d17b06cc..693fd6bc336fd3 100644 --- a/metadata-ingestion/tests/unit/glue/test_glue_source.py +++ b/metadata-ingestion/tests/unit/glue/test_glue_source.py @@ -34,7 +34,6 @@ run_and_get_pipeline, validate_all_providers_have_committed_successfully, ) -from tests.test_helpers.type_helpers import PytestConfig from tests.unit.glue.test_glue_source_stubs import ( databases_1, databases_2, @@ -174,7 +173,7 @@ def test_column_type(hive_column_type: str, expected_type: Type) -> None: @freeze_time(FROZEN_TIME) def test_glue_ingest( tmp_path: Path, - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, platform_instance: str, mce_file: str, mce_golden_file: str, @@ -410,7 +409,7 @@ def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): def test_glue_with_delta_schema_ingest( tmp_path: Path, - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, ) -> None: glue_source_instance = glue_source( platform_instance="delta_platform_instance", @@ -446,7 +445,7 @@ def test_glue_with_delta_schema_ingest( def test_glue_with_malformed_delta_schema_ingest( tmp_path: Path, - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, ) -> None: glue_source_instance = glue_source( platform_instance="delta_platform_instance", @@ -489,7 +488,7 @@ def test_glue_with_malformed_delta_schema_ingest( @freeze_time(FROZEN_TIME) def test_glue_ingest_include_table_lineage( tmp_path: Path, - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, mock_datahub_graph_instance: DataHubGraph, platform_instance: str, mce_file: str, @@ -584,7 +583,7 @@ def test_glue_ingest_include_table_lineage( @freeze_time(FROZEN_TIME) def test_glue_ingest_include_column_lineage( tmp_path: Path, - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, mock_datahub_graph_instance: DataHubGraph, platform_instance: str, mce_file: str, @@ -684,7 +683,7 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: @freeze_time(FROZEN_TIME) def test_glue_ingest_with_profiling( tmp_path: Path, - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, ) -> None: glue_source_instance = glue_source_with_profiling() mce_file = "glue_mces.json" diff --git a/metadata-ingestion/tests/unit/redshift/test_redshift_source.py b/metadata-ingestion/tests/unit/redshift/test_redshift_source.py index 8198caf50df7f4..f016312dfe47fb 100644 --- a/metadata-ingestion/tests/unit/redshift/test_redshift_source.py +++ b/metadata-ingestion/tests/unit/redshift/test_redshift_source.py @@ -1,15 +1,15 @@ from typing import Iterable -from datahub.emitter.mcp import ( - MetadataChangeProposalClass, - MetadataChangeProposalWrapper, -) +from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.redshift.config import RedshiftConfig from datahub.ingestion.source.redshift.redshift import RedshiftSource from datahub.ingestion.source.redshift.redshift_schema import RedshiftTable -from datahub.metadata.schema_classes import MetadataChangeEventClass +from datahub.metadata.schema_classes import ( + MetadataChangeEventClass, + MetadataChangeProposalClass, +) def redshift_source_setup(custom_props_flag: bool) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py index 727f2b10511b5e..a131ac9ce2a1bc 100644 --- a/metadata-ingestion/tests/unit/serde/test_serde.py +++ b/metadata-ingestion/tests/unit/serde/test_serde.py @@ -19,7 +19,6 @@ from datahub.metadata.schemas import getMetadataChangeEventSchema from tests.test_helpers import mce_helpers from tests.test_helpers.click_helpers import run_datahub_cmd -from tests.test_helpers.type_helpers import PytestConfig FROZEN_TIME = "2021-07-22 18:54:06" @@ -41,7 +40,7 @@ ], ) def test_serde_to_json( - pytestconfig: PytestConfig, tmp_path: pathlib.Path, json_filename: str + pytestconfig: pytest.Config, tmp_path: pathlib.Path, json_filename: str ) -> None: golden_file = pytestconfig.rootpath / json_filename output_file = tmp_path / "output.json" @@ -73,7 +72,7 @@ def test_serde_to_json( ) @freeze_time(FROZEN_TIME) def test_serde_to_avro( - pytestconfig: PytestConfig, + pytestconfig: pytest.Config, json_filename: str, ) -> None: # In this test, we want to read in from JSON -> MCE object. @@ -126,14 +125,14 @@ def test_serde_to_avro( ], ) @freeze_time(FROZEN_TIME) -def test_check_metadata_schema(pytestconfig: PytestConfig, json_filename: str) -> None: +def test_check_metadata_schema(pytestconfig: pytest.Config, json_filename: str) -> None: json_file_path = pytestconfig.rootpath / json_filename run_datahub_cmd(["check", "metadata-file", f"{json_file_path}"]) def test_check_metadata_rewrite( - pytestconfig: PytestConfig, tmp_path: pathlib.Path + pytestconfig: pytest.Config, tmp_path: pathlib.Path ) -> None: json_input = ( pytestconfig.rootpath / "tests/unit/serde/test_canonicalization_input.json" @@ -161,7 +160,7 @@ def test_check_metadata_rewrite( ], ) def test_check_mce_schema_failure( - pytestconfig: PytestConfig, json_filename: str + pytestconfig: pytest.Config, json_filename: str ) -> None: json_file_path = pytestconfig.rootpath / json_filename diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py index 744d43373a0a1f..4e8ba8aa6b7770 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_utils.py @@ -4,11 +4,9 @@ import pytest import sqlglot +from datahub.sql_parsing.query_types import get_query_type_of_sql from datahub.sql_parsing.sql_parsing_common import QueryType -from datahub.sql_parsing.sqlglot_lineage import ( - _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT, - get_query_type_of_sql, -) +from datahub.sql_parsing.sqlglot_lineage import _UPDATE_ARGS_NOT_SUPPORTED_BY_SELECT from datahub.sql_parsing.sqlglot_utils import ( generalize_query, generalize_query_fast, diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py index 66564dc856abae..96ab8f7a01a386 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stateful_ingestion.py @@ -10,7 +10,8 @@ from datahub.api.entities.dataprocess.dataprocess_instance import DataProcessInstance from datahub.configuration.common import AllowDenyPattern -from datahub.configuration.source_common import DEFAULT_ENV, DatasetSourceConfigMixin +from datahub.configuration.source_common import DatasetSourceConfigMixin +from datahub.emitter.mce_builder import DEFAULT_ENV from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceReport From 5519a330e2d0f2cb3a3c9a75cdcdefaff2673078 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Nov 2024 13:33:54 -0800 Subject: [PATCH 036/174] chore(ingest): bump black (#11898) --- .../examples/cli_usage/gen_schemas.py | 1 - metadata-ingestion/setup.py | 21 ++++++++++++------- .../structuredproperties.py | 1 - .../src/datahub/ingestion/fs/s3_fs.py | 1 - .../glossary/classification_mixin.py | 2 -- .../source/bigquery_v2/bigquery_config.py | 1 - .../source/bigquery_v2/bigquery_schema_gen.py | 4 ---- .../ingestion/source/bigquery_v2/lineage.py | 2 -- .../source/bigquery_v2/queries_extractor.py | 4 ---- .../ingestion/source/cassandra/cassandra.py | 1 - .../source/confluent_schema_registry.py | 1 - .../source/data_lake_common/config.py | 1 - .../source/datahub/datahub_source.py | 1 - .../ingestion/source/dremio/dremio_api.py | 2 -- .../ingestion/source/dremio/dremio_config.py | 1 - .../ingestion/source/dremio/dremio_source.py | 1 - .../ingestion/source/dynamodb/dynamodb.py | 1 - .../source/gc/execution_request_cleanup.py | 1 - .../ingestion/source/ge_profiling_config.py | 1 - .../ingestion/source/looker/looker_common.py | 5 ----- .../ingestion/source/looker/looker_source.py | 2 -- .../source/looker/looker_template_language.py | 3 --- .../source/looker/lookml_concept_context.py | 2 -- .../ingestion/source/looker/lookml_source.py | 3 --- .../ingestion/source/looker/view_upstream.py | 3 --- .../ingestion/source/metadata/lineage.py | 1 - .../src/datahub/ingestion/source/nifi.py | 1 - .../powerbi/m_query/native_sql_parser.py | 1 - .../source/powerbi/m_query/parser.py | 1 - .../source/powerbi/m_query/resolver.py | 6 ------ .../powerbi/rest_api_wrapper/data_resolver.py | 2 -- .../powerbi/rest_api_wrapper/powerbi_api.py | 2 -- .../powerbi_report_server/report_server.py | 1 - .../ingestion/source/redshift/redshift.py | 1 - .../src/datahub/ingestion/source/s3/source.py | 1 - .../src/datahub/ingestion/source/sac/sac.py | 1 - .../source/snowflake/snowflake_lineage_v2.py | 1 - .../ingestion/source/sql/cockroachdb.py | 1 - .../datahub/ingestion/source/sql/oracle.py | 6 ------ .../source/state/entity_removal_state.py | 6 +++++- .../ingestion/source/tableau/tableau.py | 1 - .../transformer/add_dataset_dataproduct.py | 1 - .../ingestion/transformer/add_dataset_tags.py | 1 - .../extract_ownership_from_tags.py | 1 - .../transformer/replace_external_url.py | 1 - .../ingestion/transformer/tags_to_terms.py | 1 - .../snowflake/metric_sql_generator.py | 1 - .../src/datahub/specific/dashboard.py | 1 - .../sql_parsing/sql_parsing_aggregator.py | 1 - .../datahub/sql_parsing/sqlglot_lineage.py | 1 - .../sql_parsing/tool_meta_extractor.py | 1 - .../src/datahub/testing/mcp_diff.py | 2 +- .../src/datahub/utilities/mapping.py | 2 -- .../utilities/threaded_iterator_executor.py | 1 - .../integration/azure_ad/test_azure_ad.py | 1 - .../bigquery_v2/test_bigquery_queries.py | 1 - .../tests/integration/dremio/test_dremio.py | 1 - .../tests/integration/looker/test_looker.py | 1 - .../tests/integration/lookml/test_lookml.py | 1 - .../tests/integration/okta/test_okta.py | 3 --- .../tests/integration/oracle/common.py | 1 - .../integration/powerbi/test_m_parser.py | 2 -- .../tests/integration/powerbi/test_powerbi.py | 9 -------- .../integration/qlik_sense/test_qlik_sense.py | 2 -- .../tests/integration/sigma/test_sigma.py | 3 --- .../tests/integration/snowflake/common.py | 1 - .../tableau/test_tableau_ingest.py | 2 -- .../unity/test_unity_catalog_ingest.py | 1 - .../performance/snowflake/test_snowflake.py | 1 - .../entities/common/test_serialized_value.py | 3 --- .../test_platform_resource.py | 1 - .../test_incremental_lineage_helper.py | 1 - .../unit/bigquery/test_bigquery_lineage.py | 2 -- .../bigquery/test_bigqueryv2_usage_source.py | 1 - .../unit/redshift/test_redshift_lineage.py | 1 - .../unit/sql_parsing/test_sql_aggregator.py | 1 - .../test_stale_entity_removal_handler.py | 1 - .../tests/unit/test_cassandra_source.py | 1 - 78 files changed, 19 insertions(+), 137 deletions(-) diff --git a/metadata-ingestion/examples/cli_usage/gen_schemas.py b/metadata-ingestion/examples/cli_usage/gen_schemas.py index 2fd4683347a3ba..80b2c6712977ad 100644 --- a/metadata-ingestion/examples/cli_usage/gen_schemas.py +++ b/metadata-ingestion/examples/cli_usage/gen_schemas.py @@ -28,7 +28,6 @@ class CorpGroupFile(BaseModel): with open("user/user.dhub.yaml_schema.json", "w") as fp: - fp.write(json.dumps(CorpUserFile.schema(), indent=4)) with open("group/group.dhub.yaml_schema.json", "w") as fp: diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 2469af74b03343..c6530c51c949d0 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -591,22 +591,26 @@ "memray", } -base_dev_requirements = { - *base_requirements, - *framework_common, - *mypy_stubs, - *s3_base, +lint_requirements = { # This is pinned only to avoid spurious errors in CI. # We should make an effort to keep it up to date. - "black==22.12.0", - "coverage>=5.1", - "faker>=18.4.0", + "black==23.3.0", "flake8>=6.0.0", "flake8-tidy-imports>=4.3.0", "flake8-bugbear==23.3.12", "isort>=5.7.0", "mypy==1.10.1", +} + +base_dev_requirements = { + *base_requirements, + *framework_common, + *mypy_stubs, + *s3_base, + *lint_requirements, *test_api_requirements, + "coverage>=5.1", + "faker>=18.4.0", "pytest-asyncio>=0.16.0", "pytest-cov>=2.8.1", "pytest-random-order~=1.1.0", @@ -931,6 +935,7 @@ ), "cloud": ["acryl-datahub-cloud"], "dev": list(dev_requirements), + "lint": list(lint_requirements), "testing-utils": list(test_api_requirements), # To import `datahub.testing` "integration-tests": list(full_test_dev_requirements), "debug": list(debug_requirements), diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py index b48c655015d825..56e02e4329055a 100644 --- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py +++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py @@ -190,7 +190,6 @@ def create(file: str, graph: Optional[DataHubGraph] = None) -> None: @classmethod def from_datahub(cls, graph: DataHubGraph, urn: str) -> "StructuredProperties": - with StructuredPropertiesConfig.use_graph(graph): structured_property: Optional[ StructuredPropertyDefinitionClass diff --git a/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py b/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py index a135b7b6ce8375..9c34c4f83b0a93 100644 --- a/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py +++ b/metadata-ingestion/src/datahub/ingestion/fs/s3_fs.py @@ -32,7 +32,6 @@ def __str__(self): class S3ListIterator(Iterator): - MAX_KEYS = 1000 def __init__( diff --git a/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py b/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py index 1d381acbf3dbe9..98c43079a3bc15 100644 --- a/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py +++ b/metadata-ingestion/src/datahub/ingestion/glossary/classification_mixin.py @@ -33,7 +33,6 @@ @dataclass class ClassificationReportMixin: - num_tables_fetch_sample_values_failed: int = 0 num_tables_classification_attempted: int = 0 @@ -112,7 +111,6 @@ def classify_schema_fields( schema_metadata: SchemaMetadata, sample_data: Union[Dict[str, list], Callable[[], Dict[str, list]]], ) -> None: - if not isinstance(sample_data, Dict): try: # TODO: In future, sample_data fetcher can be lazily called if classification diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index ad293c702a5205..4af41921c9fa3c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -374,7 +374,6 @@ class BigQueryV2Config( StatefulProfilingConfigMixin, ClassificationSourceConfigMixin, ): - include_schema_metadata: bool = Field( default=True, description="Whether to ingest the BigQuery schema, i.e. projects, schemas, tables, and views.", diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py index 345467ab76c866..6f3008ccfd6923 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py @@ -356,7 +356,6 @@ def _process_project( project_id ) except Exception as e: - if self.config.project_ids and "not enabled BigQuery." in str(e): action_mesage = ( "The project has not enabled BigQuery API. " @@ -417,7 +416,6 @@ def _process_project_datasets( bigquery_project: BigqueryProject, db_tables: Dict[str, List[BigqueryTable]], ) -> Iterable[MetadataWorkUnit]: - db_views: Dict[str, List[BigqueryView]] = {} db_snapshots: Dict[str, List[BigqueryTableSnapshot]] = {} project_id = bigquery_project.id @@ -1141,7 +1139,6 @@ def gen_schema_metadata( columns: List[BigqueryColumn], dataset_name: BigqueryTableIdentifier, ) -> MetadataWorkUnit: - foreign_keys: List[ForeignKeyConstraint] = [] # Foreign keys only make sense for tables if isinstance(table, BigqueryTable): @@ -1183,7 +1180,6 @@ def get_tables_for_dataset( ) -> Iterable[BigqueryTable]: # In bigquery there is no way to query all tables in a Project id with PerfTimer() as timer: - # PARTITIONS INFORMATION_SCHEMA view is not available for BigLake tables # based on Amazon S3 and Blob Storage data. # https://cloud.google.com/bigquery/docs/omni-introduction#limitations diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py index b542992a7924a0..321b1b6207fabf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py @@ -934,7 +934,6 @@ def gen_lineage_workunits_for_external_table( ddl: Optional[str], graph: Optional[DataHubGraph] = None, ) -> Iterable[MetadataWorkUnit]: - if not ddl: return @@ -972,7 +971,6 @@ def get_lineage_for_external_table( source_uris: List[str], graph: Optional[DataHubGraph] = None, ) -> Optional[UpstreamLineageClass]: - upstreams_list: List[UpstreamClass] = [] fine_grained_lineages: List[FineGrainedLineageClass] = [] gcs_urns: Set[str] = set() diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py index 497947abe4ef9a..91d55ad879e04a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py @@ -304,7 +304,6 @@ def get_workunits_internal( def deduplicate_queries( self, queries: FileBackedList[ObservedQuery] ) -> FileBackedDict[Dict[int, ObservedQuery]]: - # This fingerprint based deduplication is done here to reduce performance hit due to # repetitive sql parsing while adding observed query to aggregator that would otherwise # parse same query multiple times. In future, aggregator may absorb this deduplication. @@ -342,7 +341,6 @@ def deduplicate_queries( return queries_deduped def fetch_query_log(self, project: BigqueryProject) -> Iterable[ObservedQuery]: - # Multi-regions from https://cloud.google.com/bigquery/docs/locations#supported_locations regions = self.config.region_qualifiers @@ -355,7 +353,6 @@ def fetch_query_log(self, project: BigqueryProject) -> Iterable[ObservedQuery]: def fetch_region_query_log( self, project: BigqueryProject, region: str ) -> Iterable[ObservedQuery]: - # Each region needs to be a different query query_log_query = _build_enriched_query_log_query( project_id=project.id, @@ -452,7 +449,6 @@ def _build_enriched_query_log_query( start_time: datetime, end_time: datetime, ) -> str: - audit_start_time = start_time.strftime(BQ_DATETIME_FORMAT) audit_end_time = end_time.strftime(BQ_DATETIME_FORMAT) diff --git a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py index 6a5236563f48db..dcdccc08ce0483 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py +++ b/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py @@ -332,7 +332,6 @@ def _extract_columns_from_table( def _extract_views_from_keyspace( self, keyspace_name: str ) -> Iterable[MetadataWorkUnit]: - views: List[CassandraView] = self.cassandra_api.get_views(keyspace_name) for view in views: view_name: str = view.view_name diff --git a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py index 09ce8b5b05203c..ed51487ea6dab2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py @@ -371,7 +371,6 @@ def _get_schema_fields( def _get_schema_metadata( self, topic: str, platform_urn: str, is_subject: bool ) -> Optional[SchemaMetadata]: - # Process the value schema schema, fields = self._get_schema_and_fields( topic=topic, diff --git a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py index 5f88cf0234947a..ede7d3c3c56959 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/data_lake_common/config.py @@ -7,7 +7,6 @@ class PathSpecsConfigMixin(ConfigModel): - path_specs: List[PathSpec] = Field( description="List of PathSpec. See [below](#path-spec) the details about PathSpec" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py index de212ca9a67716..63cea45f75864b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py @@ -107,7 +107,6 @@ def _get_database_workunits( logger.info(f"Fetching database aspects starting from {from_createdon}") mcps = reader.get_aspects(from_createdon, self.report.stop_time) for i, (mcp, createdon) in enumerate(mcps): - if not self.urn_pattern.allowed(str(mcp.entityUrn)): continue diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py index db83dde7cf6131..7b9ccb52acbef4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py @@ -566,7 +566,6 @@ def get_all_tables_and_columns(self, containers: Deque) -> List[Dict]: return tables def validate_schema_format(self, schema): - if "." in schema: schema_path = self.get( url=f"/catalog/{self.get_dataset_id(schema=schema, dataset='')}" @@ -687,7 +686,6 @@ def traverse_path(location_id: str, entity_path: List[str]) -> List: response.get("entityType") == DremioEntityContainerType.FOLDER.value.lower() ): - containers.append( { "id": location_id, diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py index 9d6f65b95554e7..d966d575c03320 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py @@ -121,7 +121,6 @@ class DremioSourceConfig( EnvConfigMixin, PlatformInstanceConfigMixin, ): - domain: Optional[str] = Field( default=None, description="Domain for all source objects.", diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py index cd6ba441b5c93b..5b96845ec04961 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py @@ -198,7 +198,6 @@ def _build_source_map(self) -> Dict[str, Dict]: source_platform_name = source_name for mapping in self.config.source_mappings or []: - if re.search(mapping.source_name, source_type, re.IGNORECASE): source_platform_name = mapping.source_name.lower() diff --git a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py index 4f1de6fb06c695..cb3f0dd9cf29f4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py @@ -233,7 +233,6 @@ def _process_table( table_name: str, dataset_name: str, ) -> Iterable[MetadataWorkUnit]: - logger.debug(f"Processing table: {dataset_name}") table_info = dynamodb_client.describe_table(TableName=table_name)["Table"] account_id = table_info["TableArn"].split(":")[4] diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py index 570df4e99ab13d..3baf858e44cdc8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/execution_request_cleanup.py @@ -69,7 +69,6 @@ def __init__( report: DatahubExecutionRequestCleanupReport, config: Optional[DatahubExecutionRequestCleanupConfig] = None, ) -> None: - self.graph = graph self.report = report self.instance_id = int(time.time()) diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index 8b2443a589b8dc..c20506e36a844f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -95,7 +95,6 @@ class GEProfilingBaseConfig(ConfigModel): class GEProfilingConfig(GEProfilingBaseConfig): - report_dropped_profiles: bool = Field( default=False, description="Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py index 3e2872a4b5caa1..57a251ef2ed14f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py @@ -307,7 +307,6 @@ def view_fields_from_dict( type_cls: ViewFieldType, populate_sql_logic_in_descriptions: bool, ) -> "ViewField": - is_primary_key = field_dict.get("primary_key", "no") == "yes" name = field_dict["name"] @@ -929,7 +928,6 @@ def from_api( # noqa: C901 reporter: SourceReport, source_config: LookerDashboardSourceConfig, ) -> Optional["LookerExplore"]: # noqa: C901 - try: explore = client.lookml_model_explore(model, explore_name) views: Set[str] = set() @@ -987,13 +985,11 @@ def from_api( # noqa: C901 field_name_vs_raw_explore_field: Dict = {} if explore.fields is not None: - if explore.fields.dimensions is not None: for dim_field in explore.fields.dimensions: if dim_field.name is None: continue else: - field_name_vs_raw_explore_field[dim_field.name] = dim_field view_fields.append( @@ -1034,7 +1030,6 @@ def from_api( # noqa: C901 if measure_field.name is None: continue else: - field_name_vs_raw_explore_field[ measure_field.name ] = measure_field diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index e42ac7b61c1777..cd8ccb8217257c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -604,7 +604,6 @@ def _get_folder_browse_path_v2_entries( def _create_platform_instance_aspect( self, ) -> DataPlatformInstance: - assert ( self.source_config.platform_name ), "Platform name is not set in the configuration." @@ -999,7 +998,6 @@ def _gen_folder_key(self, folder_id: str) -> LookerFolderKey: def _make_dashboard_and_chart_mces( self, looker_dashboard: LookerDashboard ) -> Iterable[Union[MetadataChangeEvent, MetadataChangeProposalWrapper]]: - # Step 1: Emit metadata for each Chart inside the Dashboard. chart_events = [] for element in looker_dashboard.dashboard_elements: diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py index 1e60c08fe00c2b..6d49d57e077435 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_template_language.py @@ -55,7 +55,6 @@ def _create_new_liquid_variables_with_default( current_dict: dict = new_dict for key in keys[:-1]: - if key not in current_dict: current_dict[key] = {} @@ -392,7 +391,6 @@ def process_lookml_template_language( source_config: LookMLSourceConfig, view_lkml_file_dict: dict, ) -> None: - if "views" not in view_lkml_file_dict: return @@ -425,7 +423,6 @@ def load_and_preprocess_file( path: Union[str, pathlib.Path], source_config: LookMLSourceConfig, ) -> dict: - parsed = load_lkml(path) process_lookml_template_language( diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py index ce4a242027e11a..80be566cdcd468 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_concept_context.py @@ -320,7 +320,6 @@ def get_including_extends( self, field: str, ) -> Optional[Any]: - # According to Looker's inheritance rules, we need to merge the fields(i.e. dimensions, measures and # dimension_groups) from both the child and parent. if field in [DIMENSIONS, DIMENSION_GROUPS, MEASURES]: @@ -345,7 +344,6 @@ def _get_sql_table_name_field(self) -> Optional[str]: return self.get_including_extends(field="sql_table_name") def _is_dot_sql_table_name_present(self) -> bool: - sql_table_name: Optional[str] = self._get_sql_table_name_field() if sql_table_name is None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 3c83b8728aa6f7..c7d3724472d3c8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -144,7 +144,6 @@ def from_looker_dict( extract_col_level_lineage: bool = False, populate_sql_logic_in_descriptions: bool = False, ) -> Optional["LookerView"]: - view_name = view_context.name() logger.debug(f"Handling view {view_name} in model {model_name}") @@ -418,7 +417,6 @@ def _get_custom_properties(self, looker_view: LookerView) -> DatasetPropertiesCl def _build_dataset_mcps( self, looker_view: LookerView ) -> List[MetadataChangeProposalWrapper]: - view_urn = looker_view.id.get_urn(self.source_config) subTypeEvent = MetadataChangeProposalWrapper( @@ -502,7 +500,6 @@ def get_project_name(self, model_name: str) -> str: def get_manifest_if_present(self, folder: pathlib.Path) -> Optional[LookerManifest]: manifest_file = folder / "manifest.lkml" if manifest_file.exists(): - manifest_dict = load_and_preprocess_file( path=manifest_file, source_config=self.source_config ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py index 632d0caf712323..8cec6f2607774e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/view_upstream.py @@ -72,7 +72,6 @@ def resolve_derived_view_urn_of_col_ref( base_folder_path: str, config: LookMLSourceConfig, ) -> List[ColumnRef]: - new_column_refs: List[ColumnRef] = [] for col_ref in column_refs: if is_derived_view(col_ref.table.lower()): @@ -641,7 +640,6 @@ def create_view_upstream( ctx: PipelineContext, reporter: LookMLSourceReport, ) -> AbstractViewUpstream: - if view_context.is_regular_case(): return RegularViewUpstream( view_context=view_context, @@ -666,7 +664,6 @@ def create_view_upstream( view_context.is_sql_based_derived_view_without_fields_case(), ] ): - return DerivedQueryUpstreamSource( view_context=view_context, config=config, diff --git a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py index 08ed7677c7ab4c..9f96f837eb9b3a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py @@ -210,7 +210,6 @@ def _get_lineage_mcp( # extract the old lineage and save it for the new mcp if preserve_upstream: - client = get_default_graph() old_upstream_lineage = get_aspects_for_entity( diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py index 7072ebf6473df1..f55d7a883edefe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/nifi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/nifi.py @@ -464,7 +464,6 @@ def report_dropped(self, ent_name: str) -> None: @support_status(SupportStatus.CERTIFIED) @capability(SourceCapability.LINEAGE_COARSE, "Supported. See docs for limitations") class NifiSource(Source): - config: NifiSourceConfig report: NifiSourceReport diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py index 63a6073c90a1a9..8ffd54613eb380 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/native_sql_parser.py @@ -101,7 +101,6 @@ def parse_custom_sql( env: str, platform_instance: Optional[str], ) -> Optional["SqlParsingResult"]: - logger.debug("Using sqlglot_lineage to parse custom sql") logger.debug(f"Processing native query using DataHub Sql Parser = {query}") diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py index 15524137c0a85e..97698a3d0d56c1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py @@ -66,7 +66,6 @@ def get_upstream_tables( config: PowerBiDashboardSourceConfig, parameters: Dict[str, str] = {}, ) -> List[resolver.Lineage]: - if table.expression is None: logger.debug(f"There is no M-Query expression in table {table.full_name}") return [] diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py index 32de95d6bd015e..a40e67d08da5b2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py @@ -65,7 +65,6 @@ def urn_creator( server: str, qualified_table_name: str, ) -> str: - platform_detail: PlatformDetail = platform_instance_resolver.get_platform_instance( PowerBIPlatformDetail( data_platform_pair=data_platform_pair, @@ -179,7 +178,6 @@ def create_reference_table( arg_list: Tree, table_detail: Dict[str, str], ) -> Optional[ReferencedTable]: - arguments: List[str] = tree_function.strip_char_from_list( values=tree_function.remove_whitespaces_from_list( tree_function.token_values(arg_list) @@ -219,7 +217,6 @@ def create_reference_table( def parse_custom_sql( self, query: str, server: str, database: Optional[str], schema: Optional[str] ) -> Lineage: - dataplatform_tables: List[DataPlatformTable] = [] platform_detail: PlatformDetail = ( @@ -377,7 +374,6 @@ def get_argument_list(invoke_expression: Tree) -> Optional[Tree]: return argument_list def take_first_argument(self, expression: Tree) -> Optional[Tree]: - # function is not data-access function, lets process function argument first_arg_tree: Optional[Tree] = tree_function.first_arg_list_func(expression) @@ -785,7 +781,6 @@ def create_urn_using_old_parser( def create_lineage( self, data_access_func_detail: DataAccessFunctionDetail ) -> Lineage: - arguments: List[str] = tree_function.strip_char_from_list( values=tree_function.remove_whitespaces_from_list( tree_function.token_values(data_access_func_detail.arg_list) @@ -897,7 +892,6 @@ def form_qualified_table_name( table_reference: ReferencedTable, data_platform_pair: DataPlatformPair, ) -> str: - platform_detail: PlatformDetail = ( self.platform_instance_resolver.get_platform_instance( PowerBIPlatformDetail( diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py index a59d58519d6bfe..e1301edef10b84 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py @@ -439,7 +439,6 @@ def get_app( self, app_id: str, ) -> Optional[App]: - raw_app: Optional[Dict] = self._get_app( app_id=app_id, ) @@ -1062,7 +1061,6 @@ def _get_app( self, app_id: str, ) -> Optional[Dict]: - app_endpoint = self.API_ENDPOINTS[Constant.GET_WORKSPACE_APP].format( POWERBI_ADMIN_BASE_URL=DataResolverBase.ADMIN_BASE_URL, APP_ID=app_id, diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py index b49f1f09fa966e..5ae333430a78bc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py @@ -40,7 +40,6 @@ def form_full_table_name( dataset_name: str, table_name: str, ) -> str: - full_table_name: str = "{}.{}".format( dataset_name.replace(" ", "_"), table_name.replace(" ", "_") ) @@ -596,7 +595,6 @@ def _fill_metadata_from_scan_result( return workspaces def _fill_independent_datasets(self, workspace: Workspace) -> None: - reachable_datasets: List[str] = [] # Find out reachable datasets for dashboard in workspace.dashboards: diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py index 8854f9ff48348d..2a247d0c63957a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/report_server.py @@ -126,7 +126,6 @@ def log_http_error(e: BaseException, message: str) -> Any: def get_response_dict(response: requests.Response, error_message: str) -> dict: - result_dict: dict = {} try: response.raise_for_status() diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py index 76030cea984946..4bc4c1451c262f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py @@ -436,7 +436,6 @@ def get_workunits_internal(self) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit def _extract_metadata( self, connection: redshift_connector.Connection, database: str ) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: - yield from self.gen_database_container( database=database, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py index e8c70260ebc7ce..1863663f98bb24 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/s3/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/s3/source.py @@ -804,7 +804,6 @@ def get_dir_to_process( protocol: str, min: bool = False, ) -> List[str]: - # if len(path_spec.include.split("/")) == len(f"{protocol}{bucket_name}/{folder}".split("/")): # return [f"{protocol}{bucket_name}/{folder}"] diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py index 66962b5d96d389..b75f15c0ce770e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py @@ -401,7 +401,6 @@ def get_model_workunits( columns = self.get_import_data_model_columns(model_id=model.model_id) for column in columns: - schema_field = SchemaFieldClass( fieldPath=column.name, type=self.get_schema_field_data_type(column), diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index ac47abf4874499..e065e2f34bc66d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -236,7 +236,6 @@ def populate_known_query_lineage( def get_known_query_lineage( self, query: Query, dataset_name: str, db_row: UpstreamLineageEdge ) -> Optional[KnownQueryLineageInfo]: - if not db_row.UPSTREAM_TABLES: return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py b/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py index 5356cee7f6ea30..76b72d8e37f74b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py @@ -28,7 +28,6 @@ class CockroachDBConfig(PostgresConfig): @capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion") class CockroachDBSource(PostgresSource): - config: CockroachDBConfig def __init__(self, config: CockroachDBConfig, ctx: PipelineContext): diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py index bce3b29130ec9e..766b704d6ffafe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py @@ -178,7 +178,6 @@ def get_table_names(self, schema: Optional[str] = None) -> List[str]: ] def get_view_names(self, schema: Optional[str] = None) -> List[str]: - schema = self._inspector_instance.dialect.denormalize_name( schema or self.default_schema_name ) @@ -200,7 +199,6 @@ def get_view_names(self, schema: Optional[str] = None) -> List[str]: def get_columns( self, table_name: str, schema: Optional[str] = None, dblink: str = "" ) -> List[dict]: - denormalized_table_name = self._inspector_instance.dialect.denormalize_name( table_name ) @@ -344,7 +342,6 @@ def get_columns( return columns def get_table_comment(self, table_name: str, schema: Optional[str] = None) -> Dict: - denormalized_table_name = self._inspector_instance.dialect.denormalize_name( table_name ) @@ -416,7 +413,6 @@ def _get_constraint_data( def get_pk_constraint( self, table_name: str, schema: Optional[str] = None, dblink: str = "" ) -> Dict: - denormalized_table_name = self._inspector_instance.dialect.denormalize_name( table_name ) @@ -458,7 +454,6 @@ def get_pk_constraint( def get_foreign_keys( self, table_name: str, schema: Optional[str] = None, dblink: str = "" ) -> List: - denormalized_table_name = self._inspector_instance.dialect.denormalize_name( table_name ) @@ -540,7 +535,6 @@ def fkey_rec(): def get_view_definition( self, view_name: str, schema: Optional[str] = None ) -> Union[str, None]: - denormalized_view_name = self._inspector_instance.dialect.denormalize_name( view_name ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py index 318395d4e66b2a..2b10ca1fa57ed8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py @@ -146,7 +146,11 @@ def urn_count(self) -> int: def compute_percent_entities_changed( new_entities: List[str], old_entities: List[str] ) -> float: - (overlap_count, old_count, _,) = _get_entity_overlap_and_cardinalities( + ( + overlap_count, + old_count, + _, + ) = _get_entity_overlap_and_cardinalities( new_entities=new_entities, old_entities=old_entities ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index c1d899f11f2e1d..0eafdb4ad23ba0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -2117,7 +2117,6 @@ def parse_custom_sql( def _enrich_database_tables_with_parsed_schemas( self, parsing_result: SqlParsingResult ) -> None: - in_tables_schemas: Dict[ str, Set[str] ] = transform_parsing_result_to_in_tables_schemas(parsing_result) diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py index ce224bde003fd3..bb1c297513de10 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_dataproduct.py @@ -105,7 +105,6 @@ class SimpleAddDatasetDataProduct(AddDatasetDataProduct): """Transformer that adds a specified dataproduct entity for provided dataset as its asset.""" def __init__(self, config: SimpleDatasetDataProductConfig, ctx: PipelineContext): - generic_config = AddDatasetDataProductConfig( get_data_product_to_add=lambda dataset_urn: config.dataset_to_data_product_urns.get( dataset_urn diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py index ef6ef43fa2d7f3..c60f4dca28882d 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py @@ -67,7 +67,6 @@ def transform_aspect( def handle_end_of_stream( self, ) -> List[Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]]: - mcps: List[ Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass] ] = [] diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py index 245a3aa3d9db15..212e018dd64fb7 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/extract_ownership_from_tags.py @@ -105,7 +105,6 @@ def convert_tag_as_per_mapping(self, tag: str) -> str: def handle_end_of_stream( self, ) -> Sequence[Union[MetadataChangeProposalWrapper, MetadataChangeProposalClass]]: - return self.owner_mcps def transform_aspect( diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py b/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py index 57af10d1040c8a..f6847f234aefe6 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/replace_external_url.py @@ -103,7 +103,6 @@ def create( def transform_aspect( self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect] ) -> Optional[Aspect]: - in_container_properties_aspect: ContainerPropertiesClass = cast( ContainerPropertiesClass, aspect ) diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py b/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py index 338f191c0829df..7e6125079f16e3 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/tags_to_terms.py @@ -84,7 +84,6 @@ def get_tags_from_schema_metadata( def transform_aspect( self, entity_urn: str, aspect_name: str, aspect: Optional[Aspect] ) -> Optional[Aspect]: - in_glossary_terms: Optional[GlossaryTermsClass] = cast( Optional[GlossaryTermsClass], aspect ) diff --git a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py index 5b079129e0a9c5..facc7d107d1ba7 100644 --- a/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py +++ b/metadata-ingestion/src/datahub/integrations/assertion/snowflake/metric_sql_generator.py @@ -72,7 +72,6 @@ def _(self, assertion: FixedIntervalFreshnessAssertion) -> str: @metric_sql.register def _(self, assertion: RowCountTotalVolumeAssertion) -> str: - # Can not use information schema here due to error - # Data metric function body cannot refer to the non-deterministic function 'CURRENT_DATABASE_MAIN_METASTORE_ID'. diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index 8228dbc011db2f..f57df15914369c 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -433,7 +433,6 @@ def set_description(self, description: str) -> "DashboardPatchBuilder": def add_custom_properties( self, custom_properties: Optional[Dict[str, str]] = None ) -> "DashboardPatchBuilder": - if custom_properties: for key, value in custom_properties.items(): self.custom_properties_patch_helper.add_property(key, value) diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index e8a0369597d53a..360ccd7bf35073 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -831,7 +831,6 @@ def add_preparsed_query( session_has_temp_tables: bool = True, _is_internal: bool = False, ) -> None: - # Adding tool specific metadata extraction here allows it # to work for both ObservedQuery and PreparsedQuery as # add_preparsed_query it used within add_observed_query. diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index 506bd1d8c6be40..4d0f9f7d8cf59d 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -880,7 +880,6 @@ def _sqlglot_lineage_inner( default_schema: Optional[str] = None, default_dialect: Optional[str] = None, ) -> SqlParsingResult: - if not default_dialect: dialect = get_dialect(schema_resolver.platform) else: diff --git a/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py b/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py index cdd35c23e30885..0d85002776e5e2 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py +++ b/metadata-ingestion/src/datahub/sql_parsing/tool_meta_extractor.py @@ -79,7 +79,6 @@ def _extract_mode_query(self, entry: QueryLog) -> bool: return True def extract_bi_metadata(self, entry: QueryLog) -> bool: - for tool, meta_extractor in self.known_tool_extractors: try: if meta_extractor(entry): diff --git a/metadata-ingestion/src/datahub/testing/mcp_diff.py b/metadata-ingestion/src/datahub/testing/mcp_diff.py index 95b8e83c7a64a5..5e669a718e9ad3 100644 --- a/metadata-ingestion/src/datahub/testing/mcp_diff.py +++ b/metadata-ingestion/src/datahub/testing/mcp_diff.py @@ -206,7 +206,7 @@ def apply_delta(self, golden: List[Dict[str, Any]]) -> None: """ aspect_diffs = [v for d in self.aspect_changes.values() for v in d.values()] for aspect_diff in aspect_diffs: - for (_, old, new) in aspect_diff.aspects_changed.keys(): + for _, old, new in aspect_diff.aspects_changed.keys(): golden[old.delta_info.idx] = new.delta_info.original indices_to_remove = set() diff --git a/metadata-ingestion/src/datahub/utilities/mapping.py b/metadata-ingestion/src/datahub/utilities/mapping.py index 4ea42d568da635..17023c7b388e76 100644 --- a/metadata-ingestion/src/datahub/utilities/mapping.py +++ b/metadata-ingestion/src/datahub/utilities/mapping.py @@ -43,7 +43,6 @@ def _make_owner_category_list( owner_category_urn: Optional[str], owner_ids: List[str], ) -> List[Dict]: - return [ { "urn": mce_builder.make_owner_urn(owner_id, owner_type), @@ -285,7 +284,6 @@ def convert_to_aspects(self, operation_map: Dict[str, list]) -> Dict[str, Any]: aspect_map[Constants.ADD_TAG_OPERATION] = tag_aspect if Constants.ADD_OWNER_OPERATION in operation_map: - owner_aspect = OwnershipClass( owners=[ OwnerClass( diff --git a/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py b/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py index 4d328ad31c6c4a..ab8987a7d2e8b2 100644 --- a/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py +++ b/metadata-ingestion/src/datahub/utilities/threaded_iterator_executor.py @@ -19,7 +19,6 @@ def process( args_list: Iterable[Tuple[Any, ...]], max_workers: int, ) -> Generator[T, None, None]: - out_q: queue.Queue[T] = queue.Queue() def _worker_wrapper( diff --git a/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py b/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py index 7005bc2e4411bf..024bb62bbe9ce9 100644 --- a/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py +++ b/metadata-ingestion/tests/integration/azure_ad/test_azure_ad.py @@ -68,7 +68,6 @@ def run_ingest( "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", mock_datahub_graph, ) as mock_checkpoint: - mock_checkpoint.return_value = mock_datahub_graph mocked_functions_reference( diff --git a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py index ef846f698f156e..806779475dea9d 100644 --- a/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py +++ b/metadata-ingestion/tests/integration/bigquery_v2/test_bigquery_queries.py @@ -47,7 +47,6 @@ def _generate_queries_cached_file(tmp_path: Path, queries_json_path: Path) -> No @patch("google.cloud.bigquery.Client") @patch("google.cloud.resourcemanager_v3.ProjectsClient") def test_queries_ingestion(project_client, client, pytestconfig, monkeypatch, tmp_path): - test_resources_dir = pytestconfig.rootpath / "tests/integration/bigquery_v2" mcp_golden_path = f"{test_resources_dir}/bigquery_queries_mcps_golden.json" mcp_output_path = tmp_path / "bigquery_queries_mcps.json" diff --git a/metadata-ingestion/tests/integration/dremio/test_dremio.py b/metadata-ingestion/tests/integration/dremio/test_dremio.py index eb999367962817..cc3a7e19bc93e2 100644 --- a/metadata-ingestion/tests/integration/dremio/test_dremio.py +++ b/metadata-ingestion/tests/integration/dremio/test_dremio.py @@ -192,7 +192,6 @@ def create_mysql_source(headers): def upload_dataset(headers): - url = f"{DREMIO_HOST}/apiv2/source/s3/file_format/warehouse/sample.parquet" payload = {"ignoreOtherFileFormats": False, "type": "Parquet"} diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index 7238a49cb37d2f..8bbf14709ff9fb 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -1047,7 +1047,6 @@ def test_independent_soft_deleted_looks( mocked_client = mock.MagicMock() with mock.patch("looker_sdk.init40") as mock_sdk: - mock_sdk.return_value = mocked_client setup_mock_look(mocked_client) setup_mock_soft_deleted_look(mocked_client) diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index a4cfbd5eadb7f3..ab55321a4d7342 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -831,7 +831,6 @@ def test_manifest_parser(pytestconfig: pytest.Config) -> None: @freeze_time(FROZEN_TIME) def test_duplicate_field_ingest(pytestconfig, tmp_path, mock_time): - test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" mce_out_file = "duplicate_ingest_mces_output.json" diff --git a/metadata-ingestion/tests/integration/okta/test_okta.py b/metadata-ingestion/tests/integration/okta/test_okta.py index 63ef8793cadddc..10148273c93666 100644 --- a/metadata-ingestion/tests/integration/okta/test_okta.py +++ b/metadata-ingestion/tests/integration/okta/test_okta.py @@ -58,14 +58,12 @@ def run_ingest( mocked_functions_reference, recipe, ): - with patch( "datahub.ingestion.source.identity.okta.OktaClient" ) as MockClient, patch( "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", mock_datahub_graph, ) as mock_checkpoint: - mock_checkpoint.return_value = mock_datahub_graph mocked_functions_reference(MockClient=MockClient) @@ -277,7 +275,6 @@ def overwrite_group_in_mocked_data(test_resources_dir, MockClient): def _init_mock_okta_client( test_resources_dir, MockClient, mock_users_json=None, mock_groups_json=None ): - okta_users_json_file = ( test_resources_dir / "okta_users.json" if mock_users_json is None diff --git a/metadata-ingestion/tests/integration/oracle/common.py b/metadata-ingestion/tests/integration/oracle/common.py index 79dbda8c30f896..9e2cc42ef10256 100644 --- a/metadata-ingestion/tests/integration/oracle/common.py +++ b/metadata-ingestion/tests/integration/oracle/common.py @@ -33,7 +33,6 @@ def scalar(self): @dataclass class MockConstraints: - constraint_name: str = "mock constraint name" constraint_type: str = "P" local_column: str = "mock column name" diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py index f4613c524316e3..f22998b47b9008 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py +++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py @@ -712,7 +712,6 @@ def test_redshift_regular_case(): def test_redshift_native_query(): - table: powerbi_data_classes.Table = powerbi_data_classes.Table( expression=M_QUERIES[22], name="category", @@ -1101,7 +1100,6 @@ def test_double_quotes_in_alias(): @patch("datahub.ingestion.source.powerbi.m_query.parser.get_lark_parser") def test_m_query_timeout(mock_get_lark_parser): - q = 'let\n Source = Value.NativeQuery(Snowflake.Databases("0DD93C6BD5A6.snowflakecomputing.com","sales_analytics_warehouse_prod",[Role="sales_analytics_member_ad"]){[Name="SL_OPERATIONS"]}[Data], "select SALE_NO AS ""\x1b[4mSaleNo\x1b[0m""#(lf) ,CODE AS ""Code""#(lf) ,ENDDATE AS ""end_date""#(lf) from SL_OPERATIONS.SALE.REPORTS#(lf) where ENDDATE > \'2024-02-03\'", null, [EnableFolding=true]),\n #"selected Row" = Table.SelectRows(Source)\nin\n #"selected Row"' table: powerbi_data_classes.Table = powerbi_data_classes.Table( diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 0f360d44c38cbe..edde11ff87d293 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -96,7 +96,6 @@ def read_mock_data(path: Union[Path, str]) -> dict: def register_mock_api( pytestconfig: pytest.Config, request_mock: Any, override_data: Optional[dict] = None ) -> None: - default_mock_data_path = ( pytestconfig.rootpath / "tests/integration/powerbi/mock_data/default_mock_response.json" @@ -467,7 +466,6 @@ def test_scan_all_workspaces( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api(pytestconfig=pytestconfig, request_mock=requests_mock) @@ -517,7 +515,6 @@ def test_extract_reports( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - enable_logging() test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" @@ -1219,7 +1216,6 @@ def test_independent_datasets_extraction( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api( @@ -1323,7 +1319,6 @@ def test_cll_extraction( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" register_mock_api( @@ -1380,7 +1375,6 @@ def test_cll_extraction_flags( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - register_mock_api( pytestconfig=pytestconfig, request_mock=requests_mock, @@ -1392,7 +1386,6 @@ def test_cll_extraction_flags( ) with pytest.raises(Exception, match=pattern): - Pipeline.create( { "run_id": "powerbi-test", @@ -1559,7 +1552,6 @@ def test_powerbi_app_ingest( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - common_app_ingest( pytestconfig=pytestconfig, requests_mock=requests_mock, @@ -1590,7 +1582,6 @@ def test_powerbi_app_ingest_info_message( mock_time: datetime.datetime, requests_mock: Any, ) -> None: - pipeline = common_app_ingest( pytestconfig=pytestconfig, requests_mock=requests_mock, diff --git a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py index ee1aafb6cf32dc..95f096cc3def35 100644 --- a/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py +++ b/metadata-ingestion/tests/integration/qlik_sense/test_qlik_sense.py @@ -1011,7 +1011,6 @@ def default_config(): def test_qlik_sense_ingest( pytestconfig, tmp_path, requests_mock, mock_websocket_send_request ): - test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" register_mock_api(request_mock=requests_mock) @@ -1051,7 +1050,6 @@ def test_qlik_sense_ingest( def test_platform_instance_ingest( pytestconfig, tmp_path, requests_mock, mock_websocket_send_request ): - test_resources_dir = pytestconfig.rootpath / "tests/integration/qlik_sense" register_mock_api(request_mock=requests_mock) diff --git a/metadata-ingestion/tests/integration/sigma/test_sigma.py b/metadata-ingestion/tests/integration/sigma/test_sigma.py index 6c01bf6dc80fe7..19fa1448fee598 100644 --- a/metadata-ingestion/tests/integration/sigma/test_sigma.py +++ b/metadata-ingestion/tests/integration/sigma/test_sigma.py @@ -420,7 +420,6 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: @pytest.mark.integration def test_sigma_ingest(pytestconfig, tmp_path, requests_mock): - test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma" register_mock_api(request_mock=requests_mock) @@ -464,7 +463,6 @@ def test_sigma_ingest(pytestconfig, tmp_path, requests_mock): @pytest.mark.integration def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock): - test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma" register_mock_api(request_mock=requests_mock) @@ -510,7 +508,6 @@ def test_platform_instance_ingest(pytestconfig, tmp_path, requests_mock): @pytest.mark.integration def test_sigma_ingest_shared_entities(pytestconfig, tmp_path, requests_mock): - test_resources_dir = pytestconfig.rootpath / "tests/integration/sigma" override_data = { diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 8f45be96625a45..9e4bb2f0eb634f 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -441,7 +441,6 @@ def default_query_results( # noqa: C901 include_column_lineage=True, ), ): - return [ { "DOWNSTREAM_TABLE_NAME": f"TEST_DB.TEST_SCHEMA.TABLE_{op_idx}", diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 62f8f6a654b588..6c45b8a47de412 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -276,7 +276,6 @@ def mock_sdk_client( datasources_side_effect: List[dict], sign_out_side_effect: List[dict], ) -> mock.MagicMock: - mock_client = mock.Mock() mocked_metadata = mock.Mock() mocked_metadata.query.side_effect = side_effect_query_metadata_response @@ -1228,7 +1227,6 @@ def test_permission_ingestion(pytestconfig, tmp_path, mock_datahub_graph): @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_permission_mode_switched_error(pytestconfig, tmp_path, mock_datahub_graph): - with mock.patch( "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", mock_datahub_graph, diff --git a/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py b/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py index c078f1b77fd1be..b8b0563a1d24e5 100644 --- a/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py +++ b/metadata-ingestion/tests/integration/unity/test_unity_catalog_ingest.py @@ -282,7 +282,6 @@ def register_mock_data(workspace_client): def mock_hive_sql(query): - if query == "DESCRIBE EXTENDED `bronze_kambi`.`bet` betStatusId": return [ ("col_name", "betStatusId"), diff --git a/metadata-ingestion/tests/performance/snowflake/test_snowflake.py b/metadata-ingestion/tests/performance/snowflake/test_snowflake.py index 73b7790b62e9e7..5042c78c2e7b91 100644 --- a/metadata-ingestion/tests/performance/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/performance/snowflake/test_snowflake.py @@ -16,7 +16,6 @@ def run_test(): - with mock.patch("snowflake.connector.connect") as mock_connect: sf_connection = mock.MagicMock() sf_cursor = mock.MagicMock() diff --git a/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py b/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py index c9f16bbcef6fc4..a72087376b78a3 100644 --- a/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py +++ b/metadata-ingestion/tests/unit/api/entities/common/test_serialized_value.py @@ -10,7 +10,6 @@ class MyTestModel(BaseModel): def test_base_model(): - test_base_model = MyTestModel( test_string_field="test_string_field", test_int_field=42, @@ -31,7 +30,6 @@ def test_base_model(): def test_dictwrapper(): - from datahub.metadata.schema_classes import DatasetPropertiesClass dataset_properties = DatasetPropertiesClass( @@ -58,7 +56,6 @@ def test_dictwrapper(): def test_raw_dictionary(): - test_object = { "test_string_field": "test_string_field", "test_int_field": 42, diff --git a/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py b/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py index a84e373dbe72c2..6a03f511fa51c5 100644 --- a/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py +++ b/metadata-ingestion/tests/unit/api/entities/platformresource/test_platform_resource.py @@ -185,7 +185,6 @@ class TestModel(BaseModel): def test_platform_resource_filters(): - query = ( ElasticPlatformResourceQuery.create_from() .group(LogicalOperator.AND) diff --git a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py index cafca521ae0148..c5c4a378894c32 100644 --- a/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py +++ b/metadata-ingestion/tests/unit/api/source_helpers/test_incremental_lineage_helper.py @@ -104,7 +104,6 @@ def test_incremental_table_lineage(tmp_path, pytestconfig): def test_incremental_table_lineage_empty_upstreams(tmp_path, pytestconfig): - urn = make_dataset_urn(platform, "dataset1") aspect = make_lineage_aspect( "dataset1", diff --git a/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py index 415977b0f8467b..a1981ccf767916 100644 --- a/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py +++ b/metadata-ingestion/tests/unit/bigquery/test_bigquery_lineage.py @@ -144,7 +144,6 @@ def test_column_level_lineage(lineage_entries: List[QueryEvent]) -> None: def test_lineage_for_external_bq_table(mock_datahub_graph_instance): - pipeline_context = PipelineContext(run_id="bq_gcs_lineage") pipeline_context.graph = mock_datahub_graph_instance @@ -239,7 +238,6 @@ def fake_schema_metadata(entity_urn: str) -> models.SchemaMetadataClass: def test_lineage_for_external_bq_table_no_column_lineage(mock_datahub_graph_instance): - pipeline_context = PipelineContext(run_id="bq_gcs_lineage") pipeline_context.graph = mock_datahub_graph_instance diff --git a/metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py b/metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py index 63de742b201a97..3247a64631da76 100644 --- a/metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py +++ b/metadata-ingestion/tests/unit/bigquery/test_bigqueryv2_usage_source.py @@ -184,7 +184,6 @@ def test_bigquery_table_sanitasitation(): def test_unquote_and_decode_unicode_escape_seq(): - # Test with a string that starts and ends with quotes and has Unicode escape sequences input_string = '"Hello \\u003cWorld\\u003e"' expected_output = "Hello " diff --git a/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py b/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py index 2e3eb8fde1292b..941d13be0a6139 100644 --- a/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py +++ b/metadata-ingestion/tests/unit/redshift/test_redshift_lineage.py @@ -221,7 +221,6 @@ def mock_redshift_connection() -> MagicMock: def mock_graph() -> DataHubGraph: - graph = MagicMock() graph._make_schema_resolver.return_value = SchemaResolver( diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py index b1ad9eb5c15d76..2a771a9847abd8 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sql_aggregator.py @@ -958,7 +958,6 @@ def test_table_lineage_via_temp_table_disordered_add( @freeze_time(FROZEN_TIME) def test_basic_usage(pytestconfig: pytest.Config) -> None: - frozen_timestamp = parse_user_datetime(FROZEN_TIME) aggregator = SqlParsingAggregator( platform="redshift", diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py index be2d8bac12e386..b04d4b86d2e4bb 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py @@ -50,7 +50,6 @@ def test_change_percent( def test_filter_ignored_entity_types(): - assert filter_ignored_entity_types( [ "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)", diff --git a/metadata-ingestion/tests/unit/test_cassandra_source.py b/metadata-ingestion/tests/unit/test_cassandra_source.py index a4ca3a0a9ef3f6..75dedde76c7c89 100644 --- a/metadata-ingestion/tests/unit/test_cassandra_source.py +++ b/metadata-ingestion/tests/unit/test_cassandra_source.py @@ -56,7 +56,6 @@ def assert_field_paths_match( def test_cassandra_schema_conversion( schema: str, expected_field_paths: List[str] ) -> None: - schema_dict: Dict[str, List[Any]] = json.loads(schema) column_infos: List = schema_dict["column_infos"] From 3b415cde6996645e00ed42df74808f08a92cc209 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Nov 2024 13:34:47 -0800 Subject: [PATCH 037/174] refactor(ingest/snowflake): move oauth config into snowflake dir (#11888) --- .../source/snowflake/oauth_config.py} | 0 .../ingestion/source/{sql => snowflake}/oauth_generator.py | 2 +- .../ingestion/source/snowflake/snowflake_connection.py | 7 +++++-- .../tests/unit/{ => snowflake}/test_snowflake_shares.py | 0 .../tests/unit/{ => snowflake}/test_snowflake_source.py | 2 +- .../tests/unit/{ => utilities}/test_parsing_util.py | 0 6 files changed, 7 insertions(+), 4 deletions(-) rename metadata-ingestion/src/datahub/{configuration/oauth.py => ingestion/source/snowflake/oauth_config.py} (100%) rename metadata-ingestion/src/datahub/ingestion/source/{sql => snowflake}/oauth_generator.py (97%) rename metadata-ingestion/tests/unit/{ => snowflake}/test_snowflake_shares.py (100%) rename metadata-ingestion/tests/unit/{ => snowflake}/test_snowflake_source.py (99%) rename metadata-ingestion/tests/unit/{ => utilities}/test_parsing_util.py (100%) diff --git a/metadata-ingestion/src/datahub/configuration/oauth.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_config.py similarity index 100% rename from metadata-ingestion/src/datahub/configuration/oauth.py rename to metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_config.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/oauth_generator.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_generator.py similarity index 97% rename from metadata-ingestion/src/datahub/ingestion/source/sql/oauth_generator.py rename to metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_generator.py index 7231c6ef6b1df5..a2dc0118b39782 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/oauth_generator.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/oauth_generator.py @@ -8,7 +8,7 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate from pydantic.types import SecretStr -from datahub.configuration.oauth import OAuthIdentityProvider +from datahub.ingestion.source.snowflake.oauth_config import OAuthIdentityProvider logger = logging.getLogger(__name__) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py index a9f454cfd3cdb3..397606400d389c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py @@ -16,14 +16,17 @@ from datahub.configuration.common import ConfigModel, ConfigurationError, MetaError from datahub.configuration.connection_resolver import auto_connection_resolver -from datahub.configuration.oauth import OAuthConfiguration, OAuthIdentityProvider from datahub.configuration.validate_field_rename import pydantic_renamed_field from datahub.ingestion.api.closeable import Closeable from datahub.ingestion.source.snowflake.constants import ( CLIENT_PREFETCH_THREADS, CLIENT_SESSION_KEEP_ALIVE, ) -from datahub.ingestion.source.sql.oauth_generator import OAuthTokenGenerator +from datahub.ingestion.source.snowflake.oauth_config import ( + OAuthConfiguration, + OAuthIdentityProvider, +) +from datahub.ingestion.source.snowflake.oauth_generator import OAuthTokenGenerator from datahub.ingestion.source.sql.sql_config import make_sqlalchemy_uri from datahub.utilities.config_clean import ( remove_protocol, diff --git a/metadata-ingestion/tests/unit/test_snowflake_shares.py b/metadata-ingestion/tests/unit/snowflake/test_snowflake_shares.py similarity index 100% rename from metadata-ingestion/tests/unit/test_snowflake_shares.py rename to metadata-ingestion/tests/unit/snowflake/test_snowflake_shares.py diff --git a/metadata-ingestion/tests/unit/test_snowflake_source.py b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py similarity index 99% rename from metadata-ingestion/tests/unit/test_snowflake_source.py rename to metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py index 72b59a3a4e4938..161dfa2b4e78f3 100644 --- a/metadata-ingestion/tests/unit/test_snowflake_source.py +++ b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py @@ -5,7 +5,6 @@ from pydantic import ValidationError from datahub.configuration.common import AllowDenyPattern -from datahub.configuration.oauth import OAuthConfiguration from datahub.configuration.pattern_utils import UUID_REGEX from datahub.ingestion.api.source import SourceCapability from datahub.ingestion.source.snowflake.constants import ( @@ -13,6 +12,7 @@ CLIENT_SESSION_KEEP_ALIVE, SnowflakeCloudProvider, ) +from datahub.ingestion.source.snowflake.oauth_config import OAuthConfiguration from datahub.ingestion.source.snowflake.snowflake_config import ( DEFAULT_TEMP_TABLES_PATTERNS, SnowflakeV2Config, diff --git a/metadata-ingestion/tests/unit/test_parsing_util.py b/metadata-ingestion/tests/unit/utilities/test_parsing_util.py similarity index 100% rename from metadata-ingestion/tests/unit/test_parsing_util.py rename to metadata-ingestion/tests/unit/utilities/test_parsing_util.py From 42bb07a35e255f5a6eda88fa3449964e591a9f1c Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Nov 2024 13:35:01 -0800 Subject: [PATCH 038/174] fix(ingest/bigquery): increase logging in bigquery-queries extractor (#11774) --- .../src/datahub/ingestion/run/pipeline.py | 1 + .../source/bigquery_v2/queries_extractor.py | 23 ++++---- .../src/datahub/utilities/progress_timer.py | 34 ++++++++++++ .../unit/utilities/test_progress_timer.py | 53 +++++++++++++++++++ 4 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 metadata-ingestion/src/datahub/utilities/progress_timer.py create mode 100644 metadata-ingestion/tests/unit/utilities/test_progress_timer.py diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py index 2e56d5866efa89..7c3a42c3e08931 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py @@ -428,6 +428,7 @@ def create( def _time_to_print(self) -> bool: self.num_intermediate_workunits += 1 current_time = int(time.time()) + # TODO: Replace with ProgressTimer. if current_time - self.last_time_printed > _REPORT_PRINT_INTERVAL_SECONDS: # we print self.num_intermediate_workunits = 0 diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py index 91d55ad879e04a..08c9beaa73c53b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py @@ -2,7 +2,7 @@ import logging import pathlib import tempfile -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Collection, Dict, Iterable, List, Optional, TypedDict from google.cloud.bigquery import Client @@ -49,6 +49,7 @@ FileBackedDict, FileBackedList, ) +from datahub.utilities.progress_timer import ProgressTimer from datahub.utilities.time import datetime_to_ts_millis logger = logging.getLogger(__name__) @@ -270,27 +271,25 @@ def get_workunits_internal( # Preprocessing stage that deduplicates the queries using query hash per usage bucket # Note: FileBackedDict is an ordered dictionary, so the order of execution of # queries is inherently maintained - queries_deduped: FileBackedDict[Dict[int, ObservedQuery]] - queries_deduped = self.deduplicate_queries(queries) + queries_deduped: FileBackedDict[ + Dict[int, ObservedQuery] + ] = self.deduplicate_queries(queries) self.report.num_unique_queries = len(queries_deduped) logger.info(f"Found {self.report.num_unique_queries} unique queries") with self.report.audit_log_load_timer, queries_deduped: - last_log_time = datetime.now() - last_report_time = datetime.now() + log_timer = ProgressTimer(timedelta(minutes=1)) + report_timer = ProgressTimer(timedelta(minutes=5)) + for i, (_, query_instances) in enumerate(queries_deduped.items()): for query in query_instances.values(): - now = datetime.now() - if (now - last_log_time).total_seconds() >= 60: + if log_timer.should_report(): logger.info( f"Added {i} deduplicated query log entries to SQL aggregator" ) - last_log_time = now - if (now - last_report_time).total_seconds() >= 300: - if self.report.sql_aggregator: - logger.info(self.report.sql_aggregator.as_string()) - last_report_time = now + if report_timer.should_report() and self.report.sql_aggregator: + logger.info(self.report.sql_aggregator.as_string()) self.aggregator.add(query) diff --git a/metadata-ingestion/src/datahub/utilities/progress_timer.py b/metadata-ingestion/src/datahub/utilities/progress_timer.py new file mode 100644 index 00000000000000..eac62cddb55f2c --- /dev/null +++ b/metadata-ingestion/src/datahub/utilities/progress_timer.py @@ -0,0 +1,34 @@ +from datetime import datetime, timedelta, timezone + + +class ProgressTimer: + def __init__(self, report_every: timedelta, report_0: bool = False): + """A helper for reporting progress at a given time interval. + + Should be used for long-running processes that iterate over a large number of items, + but each iteration is fast. + + Args: + report_every: The time interval between progress reports. + report_0: Whether to report progress on the first iteration. + """ + + self._report_every = report_every + + if report_0: + # Use the earliest possible time to force reporting on the first iteration. + self._last_report_time = datetime.min.replace(tzinfo=timezone.utc) + else: + self._last_report_time = self._now() + + def _now(self) -> datetime: + return datetime.now(timezone.utc) + + def should_report(self) -> bool: + current_time = self._now() + + should_report = (self._last_report_time + self._report_every) <= current_time + if should_report: + self._last_report_time = current_time + + return should_report diff --git a/metadata-ingestion/tests/unit/utilities/test_progress_timer.py b/metadata-ingestion/tests/unit/utilities/test_progress_timer.py new file mode 100644 index 00000000000000..139bad371bb9f4 --- /dev/null +++ b/metadata-ingestion/tests/unit/utilities/test_progress_timer.py @@ -0,0 +1,53 @@ +from datetime import timedelta +from time import sleep + +from datahub.utilities.progress_timer import ProgressTimer + + +def test_progress_timer_basic(): + timer = ProgressTimer(report_every=timedelta(milliseconds=100)) + + # First call should not report since report_0=False by default + assert not timer.should_report() + + # Call before interval elapsed should not report + sleep(0.05) # 50ms + assert not timer.should_report() + + # Call after interval elapsed should report + sleep(0.1) # Additional 100ms + assert timer.should_report() + + # Next immediate call should not report + assert not timer.should_report() + + +def test_progress_timer_with_report_0(): + timer = ProgressTimer(report_every=timedelta(milliseconds=100), report_0=True) + + # First call should report since report_0=True + assert timer.should_report() + + # Next immediate call should not report + assert not timer.should_report() + + # Call after interval elapsed should report + sleep(0.1) # 100ms + assert timer.should_report() + + +def test_progress_timer_multiple_intervals(): + timer = ProgressTimer(report_every=timedelta(milliseconds=50)) + + # First call should not report + assert not timer.should_report() + + # Check multiple intervals + sleep(0.06) # 60ms - should report + assert timer.should_report() + + sleep(0.02) # 20ms - should not report + assert not timer.should_report() + + sleep(0.05) # 50ms - should report + assert timer.should_report() From 7e9fa815807e13d96ac6fb7bd79b9851ea174d1b Mon Sep 17 00:00:00 2001 From: "Albert T. Wong" Date: Wed, 20 Nov 2024 13:35:50 -0800 Subject: [PATCH 039/174] Update the AWS instructions with EBS CSI and IAM policy instructions (#11872) Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- docs/deploy/aws.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index 49b0ea1d69ae19..c625ba26a3865a 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -53,7 +53,18 @@ ip-192-168-64-56.us-west-2.compute.internal Ready 3h v1.18.9-ek ip-192-168-8-126.us-west-2.compute.internal Ready 3h v1.18.9-eks-d1db3c ``` -Once your cluster is running, make sure to install the EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes. [add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) +### Install EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes + +Once your cluster is running, make sure to install the EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes. [add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html). By default Core DNS and VPC CNI plugins are installed. You need to manually install the EBS CSI driver. It show look this in your console when you are done. + +![Screenshot 2024-11-15 at 4 42 09 PM](https://github.com/user-attachments/assets/5a9a2af0-e804-4896-85bb-dc5834208719) + +### Add the AmazonEBSCSIDriverPolicy role to the EKS node group + +Next is to add the AmazonEBSCSIDriverPolicy role to the EKS node group. You will from the EKS Node group by going to the Compute tab in your EKS cluster and clicking on the IAM entry for the EKS node group. Add the AmazonEBSCSIDriverPolicy policy. + +![Screenshot 2024-11-15 at 4 42 29 PM](https://github.com/user-attachments/assets/8971c8d6-8543-408b-9a07-814aacb2532d) +![Screenshot 2024-11-15 at 4 42 46 PM](https://github.com/user-attachments/assets/397f9131-5f13-4d9f-a664-9921d9bbf44e) ## Setup DataHub using Helm From 37bb0c9eae507b6c552c073430cb07a35568f81f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 20 Nov 2024 13:48:56 -0800 Subject: [PATCH 040/174] fix(ingest/sql): disable patch checker (#11910) --- metadata-ingestion/src/datahub/utilities/is_pytest.py | 3 ++- metadata-ingestion/tests/conftest.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/utilities/is_pytest.py b/metadata-ingestion/src/datahub/utilities/is_pytest.py index 68bb1b285a50e9..572b4bf5356220 100644 --- a/metadata-ingestion/src/datahub/utilities/is_pytest.py +++ b/metadata-ingestion/src/datahub/utilities/is_pytest.py @@ -1,5 +1,6 @@ +import os import sys def is_pytest_running() -> bool: - return "pytest" in sys.modules + return "pytest" in sys.modules and os.environ.get("DATAHUB_TEST_MODE") == "1" diff --git a/metadata-ingestion/tests/conftest.py b/metadata-ingestion/tests/conftest.py index db025e7f806c06..4685faabfcb285 100644 --- a/metadata-ingestion/tests/conftest.py +++ b/metadata-ingestion/tests/conftest.py @@ -7,6 +7,7 @@ import pytest os.environ["DATAHUB_SUPPRESS_LOGGING_MANAGER"] = "1" +os.environ["DATAHUB_TEST_MODE"] = "1" # Enable debug logging. logging.getLogger().setLevel(logging.DEBUG) From e4d010d7571f70c752635ac5692dff2d1a9b133c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20L=C3=BCdin?= <13187726+Masterchen09@users.noreply.github.com> Date: Thu, 21 Nov 2024 01:05:50 +0100 Subject: [PATCH 041/174] docs(ingest/sac): add additional permission for SAP Analytics Cloud source to docs (#11903) --- metadata-ingestion/docs/sources/sac/sac_pre.md | 1 + 1 file changed, 1 insertion(+) diff --git a/metadata-ingestion/docs/sources/sac/sac_pre.md b/metadata-ingestion/docs/sources/sac/sac_pre.md index c62cd81fa27534..624eb61f716f92 100644 --- a/metadata-ingestion/docs/sources/sac/sac_pre.md +++ b/metadata-ingestion/docs/sources/sac/sac_pre.md @@ -4,6 +4,7 @@ - Purpose: API Access - Access: + - Story Listing - Data Import Service - Authorization Grant: Client Credentials From 7bdd0a801602fab3702b16afa766683d406ecdf5 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 21 Nov 2024 00:19:41 -0800 Subject: [PATCH 042/174] chore(ingest): always use urn creation helpers (#11911) --- metadata-ingestion/scripts/modeldocgen.py | 10 +++++++--- .../src/datahub/ingestion/source/sql/sql_common.py | 5 +++-- metadata-ingestion/tests/unit/test_sql_common.py | 12 ++++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/metadata-ingestion/scripts/modeldocgen.py b/metadata-ingestion/scripts/modeldocgen.py index ee5f06cb801baa..998947e5caa954 100644 --- a/metadata-ingestion/scripts/modeldocgen.py +++ b/metadata-ingestion/scripts/modeldocgen.py @@ -14,7 +14,11 @@ import click from datahub.configuration.common import ConfigEnum, PermissiveConfigModel -from datahub.emitter.mce_builder import make_data_platform_urn, make_dataset_urn +from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataset_urn, + make_schema_field_urn, +) from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.rest_emitter import DatahubRestEmitter from datahub.ingestion.api.common import PipelineContext, RecordEnvelope @@ -442,10 +446,10 @@ def strip_types(field_path: str) -> str: name=relnship_name, foreignDataset=foreign_dataset_urn, foreignFields=[ - f"urn:li:schemaField:({foreign_dataset_urn}, urn)" + make_schema_field_urn(foreign_dataset_urn, "urn") ], sourceFields=[ - f"urn:li:schemaField:({source_dataset_urn},{f_field.fieldPath})" + make_schema_field_urn(source_dataset_urn, f_field.fieldPath) ], ) foreign_keys.append(fkey) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 238fd88f1c9509..e5779791ed4120 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -32,6 +32,7 @@ make_data_platform_urn, make_dataplatform_instance_urn, make_dataset_urn_with_platform_instance, + make_schema_field_urn, make_tag_urn, ) from datahub.emitter.mcp import MetadataChangeProposalWrapper @@ -669,7 +670,7 @@ def get_foreign_key_metadata( ) source_fields = [ - f"urn:li:schemaField:({dataset_urn},{f})" + make_schema_field_urn(dataset_urn, f) for f in fk_dict["constrained_columns"] ] foreign_dataset = make_dataset_urn_with_platform_instance( @@ -679,7 +680,7 @@ def get_foreign_key_metadata( env=self.config.env, ) foreign_fields = [ - f"urn:li:schemaField:({foreign_dataset},{f})" + make_schema_field_urn(foreign_dataset, f) for f in fk_dict["referred_columns"] ] diff --git a/metadata-ingestion/tests/unit/test_sql_common.py b/metadata-ingestion/tests/unit/test_sql_common.py index a98bf641711220..cfb8f55bd977f7 100644 --- a/metadata-ingestion/tests/unit/test_sql_common.py +++ b/metadata-ingestion/tests/unit/test_sql_common.py @@ -38,7 +38,7 @@ def test_generate_foreign_key(): "referred_columns": ["test_referred_column"], # type: ignore } foreign_key = source.get_foreign_key_metadata( - dataset_urn="test_urn", + dataset_urn="urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD)", schema="test_schema", fk_dict=fk_dict, inspector=mock.Mock(), @@ -48,7 +48,9 @@ def test_generate_foreign_key(): assert [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_referred_schema.test_table,PROD),test_referred_column)" ] == foreign_key.foreignFields - assert ["urn:li:schemaField:(test_urn,test_column)"] == foreign_key.sourceFields + assert [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD),test_column)" + ] == foreign_key.sourceFields def test_use_source_schema_for_foreign_key_if_not_specified(): @@ -60,7 +62,7 @@ def test_use_source_schema_for_foreign_key_if_not_specified(): "referred_columns": ["test_referred_column"], # type: ignore } foreign_key = source.get_foreign_key_metadata( - dataset_urn="test_urn", + dataset_urn="urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD)", schema="test_schema", fk_dict=fk_dict, inspector=mock.Mock(), @@ -70,7 +72,9 @@ def test_use_source_schema_for_foreign_key_if_not_specified(): assert [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.test_table,PROD),test_referred_column)" ] == foreign_key.foreignFields - assert ["urn:li:schemaField:(test_urn,test_column)"] == foreign_key.sourceFields + assert [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:TEST,test_schema.base_urn,PROD),test_column)" + ] == foreign_key.sourceFields PLATFORM_FROM_SQLALCHEMY_URI_TEST_CASES: Dict[str, str] = { From 05c99abb4a075e0568999bf7c3f4f9b4ee4a6497 Mon Sep 17 00:00:00 2001 From: kevinkarchacryl Date: Thu, 21 Nov 2024 09:20:04 -0500 Subject: [PATCH 043/174] chore: update contributors list (#11915) --- .github/workflows/pr-labeler.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 1ae3edae7aa90b..7da20ece44f6d6 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -45,7 +45,8 @@ jobs: "Salman-Apptware", "mayurinehate", "noggi", - "skrydal" + "skrydal", + "kevinkarchacryl" ]'), github.actor ) From 98544aaa6a1dc85dab857a247412114c58d8ba2c Mon Sep 17 00:00:00 2001 From: Pinaki Bhattacharjee Date: Thu, 21 Nov 2024 23:01:04 +0530 Subject: [PATCH 044/174] fix(ts): Suppress ts errors on Editor.tsx (#11275) --- .../Documentation/components/editor/Editor.tsx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx index fe2a8c51f9377b..8ee0f637094d6d 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/editor/Editor.tsx @@ -50,26 +50,26 @@ export const Editor = forwardRef((props: EditorProps, ref) => { const { manager, state, getContext } = useRemirror({ extensions: () => [ new BlockquoteExtension(), - new BoldExtension(), - new BulletListExtension(), + new BoldExtension({}), + new BulletListExtension({}), new CodeBlockExtension({ syntaxTheme: 'base16_ateliersulphurpool_light' }), new CodeExtension(), - new DataHubMentionsExtension(), - new DropCursorExtension(), + new DataHubMentionsExtension({}), + new DropCursorExtension({}), new HardBreakExtension(), - new HeadingExtension(), - new HistoryExtension(), - new HorizontalRuleExtension(), + new HeadingExtension({}), + new HistoryExtension({}), + new HorizontalRuleExtension({}), new ImageExtension({ enableResizing: !readOnly }), new ItalicExtension(), new LinkExtension({ autoLink: true, defaultTarget: '_blank' }), - new ListItemExtension(), + new ListItemExtension({}), new MarkdownExtension({ htmlSanitizer: DOMPurify.sanitize, htmlToMarkdown, markdownToHtml }), new OrderedListExtension(), new UnderlineExtension(), new StrikeExtension(), new TableExtension({ resizable: false }), - ...(readOnly ? [] : [new HistoryExtension()]), + ...(readOnly ? [] : [new HistoryExtension({})]), ], content, stringHandler: 'markdown', From 87ee6747d42a8ddf4914d11ee58f7442610b0ea1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:37:53 -0600 Subject: [PATCH 045/174] chore(deps): bump cross-spawn from 7.0.3 to 7.0.6 in /smoke-test/tests/cypress (#11890) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- smoke-test/tests/cypress/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/smoke-test/tests/cypress/yarn.lock b/smoke-test/tests/cypress/yarn.lock index 2433e9f8fae08e..c6116609b11467 100644 --- a/smoke-test/tests/cypress/yarn.lock +++ b/smoke-test/tests/cypress/yarn.lock @@ -510,9 +510,9 @@ core-util-is@1.0.2: integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= cross-spawn@^7.0.0, cross-spawn@^7.0.2: - version "7.0.3" - resolved "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" From b5d5db3fbb7fd9ed4f6b49458c628b2dd513195e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:33:10 -0600 Subject: [PATCH 046/174] chore(deps): bump cross-spawn from 7.0.3 to 7.0.6 in /docs-website (#11919) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs-website/yarn.lock | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs-website/yarn.lock b/docs-website/yarn.lock index 4e457abd50af7a..9c82b27c3b61f3 100644 --- a/docs-website/yarn.lock +++ b/docs-website/yarn.lock @@ -1827,7 +1827,7 @@ "@docusaurus/theme-search-algolia" "2.4.3" "@docusaurus/types" "2.4.3" -"@docusaurus/react-loadable@5.5.2", "react-loadable@npm:@docusaurus/react-loadable@5.5.2": +"@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -4757,9 +4757,9 @@ cross-fetch@^3.1.5: node-fetch "^2.6.12" cross-spawn@^7.0.3: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" @@ -9713,6 +9713,14 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" +"react-loadable@npm:@docusaurus/react-loadable@5.5.2": + version "5.5.2" + resolved "https://registry.yarnpkg.com/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz#81aae0db81ecafbdaee3651f12804580868fa6ce" + integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== + dependencies: + "@types/react" "*" + prop-types "^15.6.2" + react-markdown@^8.0.6: version "8.0.7" resolved "https://registry.yarnpkg.com/react-markdown/-/react-markdown-8.0.7.tgz#c8dbd1b9ba5f1c5e7e5f2a44de465a3caafdf89b" From 1bfd4ee1d47d344e1bd2970e9b214a6b3d77f41f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 21 Nov 2024 17:16:04 -0800 Subject: [PATCH 047/174] feat(ingest): handle mssql casing issues in lineage (#11920) --- .../datahub/sql_parsing/sql_parsing_common.py | 6 + .../datahub/sql_parsing/sqlglot_lineage.py | 56 +- .../src/datahub/sql_parsing/sqlglot_utils.py | 2 +- .../integration/powerbi/golden_test_cll.json | 26 + .../golden_mces_mssql_no_db_to_file.json | 862 +++++++++++------- .../golden_mces_mssql_no_db_with_filter.json | 528 +++++++---- .../golden_mces_mssql_to_file.json | 528 +++++++---- ...golden_mces_mssql_with_lower_case_urn.json | 558 ++++++++---- .../integration/sql_server/setup/setup.sql | 2 + .../goldens/test_mssql_casing_resolver.json | 82 ++ .../goldens/test_mssql_select_into.json | 48 + .../unit/sql_parsing/test_sqlglot_lineage.py | 44 + 12 files changed, 1947 insertions(+), 795 deletions(-) create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json create mode 100644 metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py index a1b850b56677a4..ec7dbc8251b200 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_common.py @@ -21,6 +21,9 @@ # See more below: # https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/acreldb/n0ejgx4895bofnn14rlguktfx5r3.htm "teradata", + # For SQL server, the default collation rules mean that all identifiers (schema, table, column names) + # are case preserving but case insensitive. + "mssql", } DIALECTS_WITH_DEFAULT_UPPERCASE_COLS = { # In some dialects, column identifiers are effectively case insensitive @@ -28,6 +31,9 @@ # automatically lowercase unquoted identifiers. "snowflake", } +assert DIALECTS_WITH_DEFAULT_UPPERCASE_COLS.issubset( + DIALECTS_WITH_CASE_INSENSITIVE_COLS +) class QueryType(enum.Enum): diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index 4d0f9f7d8cf59d..9adb792a4be518 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -5,7 +5,7 @@ import logging import traceback from collections import defaultdict -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Set, Tuple, TypeVar, Union import pydantic.dataclasses import sqlglot @@ -873,6 +873,49 @@ def _translate_internal_column_lineage( ) +_StrOrNone = TypeVar("_StrOrNone", str, Optional[str]) + + +def _normalize_db_or_schema( + db_or_schema: _StrOrNone, + dialect: sqlglot.Dialect, +) -> _StrOrNone: + if db_or_schema is None: + return None + + # In snowflake, table identifiers must be uppercased to match sqlglot's behavior. + if is_dialect_instance(dialect, "snowflake"): + return db_or_schema.upper() + + # In mssql, table identifiers must be lowercased. + elif is_dialect_instance(dialect, "mssql"): + return db_or_schema.lower() + + return db_or_schema + + +def _simplify_select_into(statement: sqlglot.exp.Expression) -> sqlglot.exp.Expression: + """ + Check if the expression is a SELECT INTO statement. If so, converts it into a CTAS. + Other expressions are returned as-is. + """ + + if not (isinstance(statement, sqlglot.exp.Select) and statement.args.get("into")): + return statement + + # Convert from SELECT INTO + # to CREATE TABLE AS SELECT + into_expr: sqlglot.exp.Into = statement.args["into"].pop() + into_table = into_expr.this + + create = sqlglot.exp.Create( + this=into_table, + kind="TABLE", + expression=statement, + ) + return create + + def _sqlglot_lineage_inner( sql: sqlglot.exp.ExpOrStr, schema_resolver: SchemaResolverInterface, @@ -885,12 +928,9 @@ def _sqlglot_lineage_inner( else: dialect = get_dialect(default_dialect) - if is_dialect_instance(dialect, "snowflake"): - # in snowflake, table identifiers must be uppercased to match sqlglot's behavior. - if default_db: - default_db = default_db.upper() - if default_schema: - default_schema = default_schema.upper() + default_db = _normalize_db_or_schema(default_db, dialect) + default_schema = _normalize_db_or_schema(default_schema, dialect) + if is_dialect_instance(dialect, "redshift") and not default_schema: # On Redshift, there's no "USE SCHEMA " command. The default schema # is public, and "current schema" is the one at the front of the search path. @@ -918,6 +958,8 @@ def _sqlglot_lineage_inner( # original_statement.sql(pretty=True, dialect=dialect), # ) + statement = _simplify_select_into(statement) + # Make sure the tables are resolved with the default db / schema. # This only works for Unionable statements. For other types of statements, # we have to do it manually afterwards, but that's slightly lower accuracy diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py index 5413161d0b7a3d..bd98557e08aacc 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_utils.py @@ -61,7 +61,7 @@ def is_dialect_instance( else: platforms = list(platforms) - dialects = [sqlglot.Dialect.get_or_raise(platform) for platform in platforms] + dialects = [get_dialect(platform) for platform in platforms] if any(isinstance(dialect, dialect_class.__class__) for dialect_class in dialects): return True diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 8e06ba9a80906b..5881ad5de0bcb8 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -1024,6 +1024,32 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD)", "type": "TRANSFORMED" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_director)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),cd_agent_key)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),client_manager_closing_month)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,commopsdb.dbo.v_ps_cd_retention,PROD),month_wid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV),agent_key)" + ], + "confidenceScore": 1.0 + } ] } }, diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 558548bfc7a698..4302c41140dc6c 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1943,15 +1943,13 @@ } }, { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", "changeType": "UPSERT", - "aspectName": "dataFlowInfo", + "aspectName": "container", "aspect": { "json": { - "customProperties": {}, - "externalUrl": "", - "name": "DemoData.Foo.stored_procedures" + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" } }, "systemMetadata": { @@ -1961,9 +1959,188 @@ } }, { - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", - "changeType": "UPSERT", + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "", + "name": "DemoData.Foo.stored_procedures" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "changeType": "UPSERT", "aspectName": "dataJobInfo", "aspect": { "json": { @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2300,22 +2493,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", @@ -2425,6 +2602,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", @@ -2502,10 +2695,15 @@ "entityType": "container", "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2516,17 +2714,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2612,10 +2805,15 @@ "entityType": "container", "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2626,17 +2824,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2722,10 +2915,15 @@ "entityType": "container", "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2736,17 +2934,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2832,10 +3025,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2846,17 +3044,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -2942,10 +3135,15 @@ "entityType": "container", "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -2956,17 +3154,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3052,10 +3245,15 @@ "entityType": "container", "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3066,17 +3264,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3162,10 +3355,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3176,17 +3374,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3272,10 +3465,15 @@ "entityType": "container", "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3286,17 +3484,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3382,10 +3575,15 @@ "entityType": "container", "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -3396,17 +3594,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -3488,22 +3681,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", @@ -3669,6 +3846,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", @@ -3742,22 +3935,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", @@ -4079,6 +4256,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", @@ -4156,10 +4349,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4170,17 +4368,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -4266,10 +4459,15 @@ "entityType": "container", "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4280,17 +4478,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" - } - ] + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" } }, "systemMetadata": { @@ -4376,10 +4569,15 @@ "entityType": "container", "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] } }, "systemMetadata": { @@ -4389,16 +4587,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", - "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -4472,5 +4674,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 161d40ea91d91a..0a50556edc6388 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -2400,5 +2602,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 161d40ea91d91a..0a50556edc6388 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.PersonsView", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,20 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" } ] } @@ -2400,5 +2602,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index 29124f6fc156c4..0279a94084ce56 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "ba144ff3-f6f8-4a61-a8a5-5cf1ed172738", + "job_id": "01afcab8-187c-459f-828e-727196a1832d", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-09-16 15:59:53.077000", - "date_modified": "2024-09-16 15:59:53.217000", + "date_created": "2024-11-21 21:01:26.550000", + "date_modified": "2024-11-21 21:01:26.690000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -154,6 +154,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", @@ -231,10 +247,15 @@ "entityType": "container", "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -245,17 +266,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:7da983a1581c33cce8a106587b150f02", + "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -341,10 +357,15 @@ "entityType": "container", "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -355,17 +376,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:671f67227a05c22c9fa97c27abc56820", + "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -451,10 +467,15 @@ "entityType": "container", "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -465,17 +486,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:830660638ee785d5352ca300835af7ec", + "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -561,10 +577,15 @@ "entityType": "container", "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -575,17 +596,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:e6b69ac2a511e798a89a4186881f70b8", + "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -671,10 +687,15 @@ "entityType": "container", "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -685,17 +706,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:a5b29b900882d27c0d5fb0d5ccac92a5", + "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -781,10 +797,15 @@ "entityType": "container", "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -795,17 +816,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:b6baf19c5f148fba3d3385151a8c672f", + "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -891,10 +907,15 @@ "entityType": "container", "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -905,17 +926,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:ee19bd6cf8db0a0d086fbe78f7539bf7", + "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1001,10 +1017,15 @@ "entityType": "container", "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1015,17 +1036,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:6514a64e5b04f103c9c1dd0ebe3d8b47", + "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1111,10 +1127,15 @@ "entityType": "container", "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -1125,17 +1146,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:fd80008628a03642d6e747c460a90619", + "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -1217,22 +1233,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:61332a50b978d8ca7245ddb34565d7b1", @@ -1386,6 +1386,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1459,22 +1475,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", @@ -1942,6 +1942,183 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "is_view": "True" + }, + "name": "PersonsView", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "demodata.foo.personsview", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons;\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -1973,8 +2150,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-09-16 15:59:53.010000", - "date_modified": "2024-09-16 15:59:53.010000" + "date_created": "2024-11-21 21:01:26.483000", + "date_modified": "2024-11-21 21:01:26.483000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2007,6 +2184,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2084,10 +2277,15 @@ "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2098,17 +2296,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", + "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2194,10 +2387,15 @@ "entityType": "container", "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2208,17 +2406,12 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:f84e3b6c61876e1625f9112cbc0e988f", + "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" - } - ] + "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" } }, "systemMetadata": { @@ -2304,10 +2497,15 @@ "entityType": "container", "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + } + ] } }, "systemMetadata": { @@ -2317,16 +2515,66 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:d730a6ecf30bbb41cac5df5c0014168d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "upstreamLineage", "aspect": { "json": { - "path": [ + "upstreams": [ { - "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", - "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" + ], + "confidenceScore": 1.0 } ] } diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql index 77ecabc5a3fffc..f495db3b91cfae 100644 --- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql +++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql @@ -32,6 +32,8 @@ CREATE TABLE Foo.Persons ( Age int ); GO +CREATE VIEW Foo.PersonsView AS SELECT * FROM Foo.Persons; +GO CREATE TABLE Foo.SalesReason ( TempID int NOT NULL, diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json new file mode 100644 index 00000000000000..072260f5cd1651 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_casing_resolver.json @@ -0,0 +1,82 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "6a779a57ffb2598c301606d3a7d82142a7af8b102efa55a2c9a4e960fd55ac07", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)" + ], + "out_tables": [], + "column_lineage": [ + { + "downstream": { + "table": null, + "column": "age", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Age" + } + ] + }, + { + "downstream": { + "table": null, + "column": "name", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "native_column_type": "VARCHAR(16777216)" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Name" + } + ] + }, + { + "downstream": { + "table": null, + "column": "uppercased_col", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "native_column_type": "VARCHAR(16777216)" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Uppercased_Col" + } + ] + }, + { + "downstream": { + "table": null, + "column": "count", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [] + } + ], + "debug_info": { + "confidence": 0.9, + "generalized_statement": "SELECT Age, name, UPPERCASED_COL, COUNT(*) AS Count FROM Foo.Persons GROUP BY Age" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json new file mode 100644 index 00000000000000..ff8609922a9bd8 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/goldens/test_mssql_select_into.json @@ -0,0 +1,48 @@ +{ + "query_type": "SELECT", + "query_type_props": {}, + "query_fingerprint": "d18c43ed4d7f0e303b88c40ac078c5505be473edf0136e42a77dccc090d1eb8b", + "in_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)" + ], + "out_tables": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)" + ], + "column_lineage": [ + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)", + "column": "AGE", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [ + { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)", + "column": "Age" + } + ] + }, + { + "downstream": { + "table": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)", + "column": "Count", + "column_type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "native_column_type": "INTEGER" + }, + "upstreams": [] + } + ], + "debug_info": { + "confidence": 0.9, + "generalized_statement": "SELECT age AS AGE, COUNT(*) AS Count INTO Foo.age_dist FROM Foo.Persons GROUP BY Age" + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py index 170341230205f3..2df6f6fa26cb56 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py +++ b/metadata-ingestion/tests/unit/sql_parsing/test_sqlglot_lineage.py @@ -1279,3 +1279,47 @@ def test_sqlite_attach_database() -> None: expected_file=RESOURCE_DIR / "test_sqlite_attach_database.json", allow_table_error=True, ) + + +def test_mssql_casing_resolver() -> None: + assert_sql_result( + """\ +SELECT Age, name, UPPERCASED_COL, COUNT(*) as Count +FROM Foo.Persons +GROUP BY Age +""", + dialect="mssql", + default_db="NewData", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": { + "Age": "INTEGER", + "Name": "VARCHAR(16777216)", + "Uppercased_Col": "VARCHAR(16777216)", + }, + }, + expected_file=RESOURCE_DIR / "test_mssql_casing_resolver.json", + ) + + +def test_mssql_select_into() -> None: + assert_sql_result( + """\ +SELECT age as AGE, COUNT(*) as Count +INTO Foo.age_dist +FROM Foo.Persons +GROUP BY Age +""", + dialect="mssql", + default_db="NewData", + schemas={ + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.persons,PROD)": { + "Age": "INTEGER", + "Name": "VARCHAR(16777216)", + }, + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foo.age_dist,PROD)": { + "AGE": "INTEGER", + "Count": "INTEGER", + }, + }, + expected_file=RESOURCE_DIR / "test_mssql_select_into.json", + ) From 80ef5418cf594e3830a0c57ea1203ecd3c71ff8f Mon Sep 17 00:00:00 2001 From: Andrew Sikowitz Date: Thu, 21 Nov 2024 21:23:24 -0800 Subject: [PATCH 048/174] docs(ingest): Raise error on unsupported sqlite version (#11921) Co-authored-by: Harshal Sheth --- .../src/datahub/utilities/file_backed_collections.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py index b67f9c1a9ffa15..b0f5022446de15 100644 --- a/metadata-ingestion/src/datahub/utilities/file_backed_collections.py +++ b/metadata-ingestion/src/datahub/utilities/file_backed_collections.py @@ -228,6 +228,12 @@ def __post_init__(self) -> None: else: self._conn = ConnectionWrapper() + if sqlite3.sqlite_version_info < (3, 24, 0): + # We use the ON CONFLICT clause to implement UPSERTs with sqlite. + # This was added in 3.24.0 from 2018-06-04. + # See https://www.sqlite.org/lang_conflict.html + raise RuntimeError("SQLite version 3.24.0 or later is required") + # We keep a small cache in memory to avoid having to serialize/deserialize # data from the database too often. We use an OrderedDict to build # a poor-man's LRU cache. From dac80fb7e15b65a5a730e0e36b1f8aa0311dfaed Mon Sep 17 00:00:00 2001 From: kevinkarchacryl Date: Fri, 22 Nov 2024 02:28:12 -0500 Subject: [PATCH 049/174] fix(analytics): look at userEditableInfo to populate cells (#11909) --- .../analytics/service/AnalyticsUtil.java | 82 +++++++++---- .../graphql/utils/AnalyticsUtilTest.java | 108 ++++++++++++++++++ 2 files changed, 169 insertions(+), 21 deletions(-) create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java index a17745948eb823..88ac29b72dee83 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/service/AnalyticsUtil.java @@ -1,19 +1,14 @@ package com.linkedin.datahub.graphql.analytics.service; +import static com.linkedin.metadata.Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME; +import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; import static com.linkedin.metadata.Constants.CORP_USER_INFO_ASPECT_NAME; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.dashboard.DashboardInfo; -import com.linkedin.datahub.graphql.generated.BarSegment; -import com.linkedin.datahub.graphql.generated.Cell; -import com.linkedin.datahub.graphql.generated.Entity; -import com.linkedin.datahub.graphql.generated.EntityProfileParams; -import com.linkedin.datahub.graphql.generated.LinkParams; -import com.linkedin.datahub.graphql.generated.NamedBar; -import com.linkedin.datahub.graphql.generated.Row; -import com.linkedin.datahub.graphql.generated.SearchParams; +import com.linkedin.datahub.graphql.generated.*; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.dataplatform.DataPlatformInfo; import com.linkedin.dataset.DatasetProperties; @@ -22,6 +17,7 @@ import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.client.EntityClient; import com.linkedin.glossary.GlossaryTermInfo; +import com.linkedin.identity.CorpUserEditableInfo; import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.key.GlossaryTermKey; @@ -35,6 +31,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @@ -169,36 +166,79 @@ public static void convertToUserInfoRows( final Map gmsResponseByUser = entityClient.batchGetV2( opContext, - CORP_USER_INFO_ASPECT_NAME, + CORP_USER_ENTITY_NAME, userUrns, - ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME)); - final Map urnToCorpUserInfo = + ImmutableSet.of(CORP_USER_INFO_ASPECT_NAME, CORP_USER_EDITABLE_INFO_ASPECT_NAME)); + final Stream> entityStream = gmsResponseByUser.entrySet().stream() .filter( entry -> entry.getValue() != null - && entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME)) - .collect( - Collectors.toMap( - Map.Entry::getKey, - entry -> + && (entry.getValue().getAspects().containsKey(CORP_USER_INFO_ASPECT_NAME) + || entry + .getValue() + .getAspects() + .containsKey(CORP_USER_EDITABLE_INFO_ASPECT_NAME))); + final Map> urnToCorpUserInfo = + entityStream.collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> { + CorpUserInfo userInfo = null; + CorpUserEditableInfo editableInfo = null; + try { + userInfo = new CorpUserInfo( entry .getValue() .getAspects() .get(CORP_USER_INFO_ASPECT_NAME) .getValue() - .data()))); + .data()); + } catch (Exception e) { + // nothing to do + } + try { + + editableInfo = + new CorpUserEditableInfo( + entry + .getValue() + .getAspects() + .get(CORP_USER_EDITABLE_INFO_ASPECT_NAME) + .getValue() + .data()); + } catch (Exception e) { + // nothing to do + } + + return Pair.of(userInfo, editableInfo); + })); // Populate a row with the user link, title, and email. rows.forEach( row -> { Urn urn = UrnUtils.getUrn(row.getCells().get(0).getValue()); EntityResponse response = gmsResponseByUser.get(urn); String maybeDisplayName = response != null ? getUserName(response).orElse(null) : null; - String maybeEmail = - urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getEmail() : null; - String maybeTitle = - urnToCorpUserInfo.containsKey(urn) ? urnToCorpUserInfo.get(urn).getTitle() : null; + String maybeEmail = null; + String maybeTitle = null; + if (urnToCorpUserInfo.containsKey(urn)) { + Pair pair = urnToCorpUserInfo.get(urn); + if (pair.getLeft() != null) { + CorpUserInfo userInfo = pair.getLeft(); + maybeEmail = userInfo.getEmail(); + maybeTitle = userInfo.getTitle(); + } + if (pair.getRight() != null) { + CorpUserEditableInfo userInfo = pair.getRight(); + if (maybeEmail == null) { + maybeEmail = userInfo.getEmail(); + } + if (maybeTitle == null) { + maybeTitle = userInfo.getTitle(); + } + } + } if (maybeDisplayName != null) { row.getCells().get(0).setValue(maybeDisplayName); } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java new file mode 100644 index 00000000000000..ab1140d2380315 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/AnalyticsUtilTest.java @@ -0,0 +1,108 @@ +package com.linkedin.datahub.graphql.utils; + +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.analytics.service.AnalyticsUtil; +import com.linkedin.datahub.graphql.generated.Cell; +import com.linkedin.datahub.graphql.generated.Row; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.identity.CorpUserEditableInfo; +import com.linkedin.identity.CorpUserInfo; +import com.linkedin.metadata.Constants; +import io.datahubproject.metadata.context.OperationContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class AnalyticsUtilTest { + + @Mock private OperationContext mockOpContext; + + @Mock private EntityClient mockEntityClient; + + final String TEST_CORP_USER_INFO_TEST_USER = "Corp User"; + final String TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE = "Editable Info Title"; + final String TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL = "Editable Info Email"; + + @BeforeMethod + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + @Test + public void testConvertToUserInfoRows() throws Exception { + List rows = new ArrayList<>(); + rows.add(new Row(null, Arrays.asList(new Cell("urn:li:corpuser:testuser", null, null)))); + + // create a CorpUserInfo with only display name set + CorpUserInfo corpUserInfo = new CorpUserInfo(); + corpUserInfo.setActive(true); + corpUserInfo.setDisplayName(TEST_CORP_USER_INFO_TEST_USER); + + // create an editableInfo with the email and title set + CorpUserEditableInfo corpUserEditableInfo = new CorpUserEditableInfo(); + corpUserEditableInfo.setEmail(TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL); // Overriding email + corpUserEditableInfo.setTitle(TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE); // Overriding title + + DataMap corpUserInfoDataMap = new DataMap(); + corpUserInfoDataMap.put("name", Constants.CORP_USER_INFO_ASPECT_NAME); + corpUserInfoDataMap.put("type", "VERSIONED"); + corpUserInfoDataMap.put("value", corpUserInfo.data()); + + DataMap corpUserEditableInfoDataMap = new DataMap(); + corpUserEditableInfoDataMap.put("name", Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME); + corpUserEditableInfoDataMap.put("type", "VERSIONED"); + corpUserEditableInfoDataMap.put("value", corpUserEditableInfo.data()); + + EnvelopedAspect corpUserInfoEnvelopedAspect = new EnvelopedAspect(corpUserInfoDataMap); + EnvelopedAspect corpUserEditableInfoEnvelopedAspect = + new EnvelopedAspect(corpUserEditableInfoDataMap); + + EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put(Constants.CORP_USER_INFO_ASPECT_NAME, corpUserInfoEnvelopedAspect); + aspectMap.put( + Constants.CORP_USER_EDITABLE_INFO_ASPECT_NAME, corpUserEditableInfoEnvelopedAspect); + + EntityResponse entityResponse = new EntityResponse(); + entityResponse.setAspects(aspectMap); + + Map entityResponseMap = new HashMap<>(); + Urn userUrn = UrnUtils.getUrn("urn:li:corpuser:testuser"); + entityResponseMap.put(userUrn, entityResponse); + + // method of the entity client we need to mock to retrieve the response map + when(mockEntityClient.batchGetV2( + eq(mockOpContext), eq(Constants.CORP_USER_ENTITY_NAME), anySet(), anySet())) + .thenReturn(entityResponseMap); + + // function we are testing + AnalyticsUtil.convertToUserInfoRows(mockOpContext, mockEntityClient, rows); + + Row updatedRow = rows.get(0); + List updatedCells = updatedRow.getCells(); + + // asserting that the display user is from CorpUserInfo and email, title are from EditableInfo + assertEquals(updatedCells.get(0).getValue(), TEST_CORP_USER_INFO_TEST_USER); + assertEquals( + updatedCells.get(1).getValue(), + TEST_CORP_USER_EDITABLE_INFO_TEST_TITLE); // Overriding title + assertEquals( + updatedCells.get(2).getValue(), + TEST_CORP_USER_EDITABLE_INFO_TEST_EMAIL); // Overriding email + } +} From 86b8175627c2335311020d3504c2d307a4badb00 Mon Sep 17 00:00:00 2001 From: sid-acryl <155424659+sid-acryl@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:08:23 +0530 Subject: [PATCH 050/174] fix(ingestion/kafka): OAuth callback execution (#11900) --- .../docs/sources/kafka/kafka.md | 22 +++++++++++ metadata-ingestion/setup.py | 4 +- .../src/datahub/configuration/kafka.py | 13 ++++++- .../configuration/kafka_consumer_config.py | 35 +++++++++++++++++ .../source/confluent_schema_registry.py | 6 ++- .../ingestion/source/kafka/__init__.py | 0 .../ingestion/source/{ => kafka}/kafka.py | 14 ++++++- .../source/{ => kafka}/kafka_connect.py | 0 .../{ => kafka}/kafka_schema_registry_base.py | 0 .../integration/kafka/kafka_to_file_oauth.yml | 20 ++++++++++ .../tests/integration/kafka/oauth.py | 14 +++++++ .../tests/integration/kafka/test_kafka.py | 39 ++++++++++++++++++- .../tests/unit/api/test_pipeline.py | 8 +++- .../unit/test_confluent_schema_registry.py | 2 +- .../tests/unit/test_kafka_source.py | 32 ++++++++------- 15 files changed, 183 insertions(+), 26 deletions(-) create mode 100644 metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/kafka/__init__.py rename metadata-ingestion/src/datahub/ingestion/source/{ => kafka}/kafka.py (97%) rename metadata-ingestion/src/datahub/ingestion/source/{ => kafka}/kafka_connect.py (100%) rename metadata-ingestion/src/datahub/ingestion/source/{ => kafka}/kafka_schema_registry_base.py (100%) create mode 100644 metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml create mode 100644 metadata-ingestion/tests/integration/kafka/oauth.py diff --git a/metadata-ingestion/docs/sources/kafka/kafka.md b/metadata-ingestion/docs/sources/kafka/kafka.md index f9b50c13f02778..525f985be4662a 100644 --- a/metadata-ingestion/docs/sources/kafka/kafka.md +++ b/metadata-ingestion/docs/sources/kafka/kafka.md @@ -102,7 +102,29 @@ source: connection: bootstrap: "broker:9092" schema_registry_url: http://localhost:8081 +``` + +### OAuth Callback +The OAuth callback function can be set up using `config.connection.consumer_config.oauth_cb`. + +You need to specify a Python function reference in the format <python-module>:<function-name>. + +For example, in the configuration `oauth:create_token`, `create_token` is a function defined in `oauth.py`, and `oauth.py` must be accessible in the PYTHONPATH. +```YAML +source: + type: "kafka" + config: + # Set the custom schema registry implementation class + schema_registry_class: "datahub.ingestion.source.confluent_schema_registry.ConfluentSchemaRegistry" + # Coordinates + connection: + bootstrap: "broker:9092" + schema_registry_url: http://localhost:8081 + consumer_config: + security.protocol: "SASL_PLAINTEXT" + sasl.mechanism: "OAUTHBEARER" + oauth_cb: "oauth:create_token" # sink configs ``` diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index c6530c51c949d0..8ae112c0ab0b2b 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -741,8 +741,8 @@ "hive = datahub.ingestion.source.sql.hive:HiveSource", "hive-metastore = datahub.ingestion.source.sql.hive_metastore:HiveMetastoreSource", "json-schema = datahub.ingestion.source.schema.json_schema:JsonSchemaSource", - "kafka = datahub.ingestion.source.kafka:KafkaSource", - "kafka-connect = datahub.ingestion.source.kafka_connect:KafkaConnectSource", + "kafka = datahub.ingestion.source.kafka.kafka:KafkaSource", + "kafka-connect = datahub.ingestion.source.kafka.kafka_connect:KafkaConnectSource", "ldap = datahub.ingestion.source.ldap:LDAPSource", "looker = datahub.ingestion.source.looker.looker_source:LookerDashboardSource", "lookml = datahub.ingestion.source.looker.lookml_source:LookMLSource", diff --git a/metadata-ingestion/src/datahub/configuration/kafka.py b/metadata-ingestion/src/datahub/configuration/kafka.py index 07e2f759bb3ff6..b8d9ff994a51ab 100644 --- a/metadata-ingestion/src/datahub/configuration/kafka.py +++ b/metadata-ingestion/src/datahub/configuration/kafka.py @@ -1,6 +1,7 @@ from pydantic import Field, validator -from datahub.configuration.common import ConfigModel +from datahub.configuration.common import ConfigModel, ConfigurationError +from datahub.configuration.kafka_consumer_config import CallableConsumerConfig from datahub.configuration.validate_host_port import validate_host_port @@ -36,6 +37,16 @@ class KafkaConsumerConnectionConfig(_KafkaConnectionConfig): description="Extra consumer config serialized as JSON. These options will be passed into Kafka's DeserializingConsumer. See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md .", ) + @validator("consumer_config") + @classmethod + def resolve_callback(cls, value: dict) -> dict: + if CallableConsumerConfig.is_callable_config(value): + try: + value = CallableConsumerConfig(value).callable_config() + except Exception as e: + raise ConfigurationError(e) + return value + class KafkaProducerConnectionConfig(_KafkaConnectionConfig): """Configuration class for holding connectivity information for Kafka producers""" diff --git a/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py new file mode 100644 index 00000000000000..d3ff5998d3e790 --- /dev/null +++ b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py @@ -0,0 +1,35 @@ +import logging +from typing import Any, Dict, Optional + +from datahub.ingestion.api.registry import import_path + +logger = logging.getLogger(__name__) + + +class CallableConsumerConfig: + CALLBACK_ATTRIBUTE: str = "oauth_cb" + + def __init__(self, config: Dict[str, Any]): + self._config = config + + self._resolve_oauth_callback() + + def callable_config(self) -> Dict[str, Any]: + return self._config + + @staticmethod + def is_callable_config(config: Dict[str, Any]) -> bool: + return CallableConsumerConfig.CALLBACK_ATTRIBUTE in config + + def get_call_back_attribute(self) -> Optional[str]: + return self._config.get(CallableConsumerConfig.CALLBACK_ATTRIBUTE) + + def _resolve_oauth_callback(self) -> None: + if not self.get_call_back_attribute(): + return + + call_back = self.get_call_back_attribute() + + assert call_back # to silent lint + # Set the callback + self._config[CallableConsumerConfig.CALLBACK_ATTRIBUTE] = import_path(call_back) diff --git a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py index ed51487ea6dab2..2b75d0dca53cb7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py +++ b/metadata-ingestion/src/datahub/ingestion/source/confluent_schema_registry.py @@ -16,8 +16,10 @@ from datahub.ingestion.extractor import protobuf_util, schema_util from datahub.ingestion.extractor.json_schema_util import JsonSchemaTranslator from datahub.ingestion.extractor.protobuf_util import ProtobufSchema -from datahub.ingestion.source.kafka import KafkaSourceConfig, KafkaSourceReport -from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase +from datahub.ingestion.source.kafka.kafka import KafkaSourceConfig, KafkaSourceReport +from datahub.ingestion.source.kafka.kafka_schema_registry_base import ( + KafkaSchemaRegistryBase, +) from datahub.metadata.com.linkedin.pegasus2avro.schema import ( KafkaSchema, SchemaField, diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py similarity index 97% rename from metadata-ingestion/src/datahub/ingestion/source/kafka.py rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py index a757250a0d6c85..06d929774240ba 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py @@ -18,6 +18,7 @@ from datahub.configuration.common import AllowDenyPattern from datahub.configuration.kafka import KafkaConsumerConnectionConfig +from datahub.configuration.kafka_consumer_config import CallableConsumerConfig from datahub.configuration.source_common import ( DatasetSourceConfigMixin, LowerCaseDatasetUrnConfigMixin, @@ -49,7 +50,9 @@ ) from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import DatasetSubTypes -from datahub.ingestion.source.kafka_schema_registry_base import KafkaSchemaRegistryBase +from datahub.ingestion.source.kafka.kafka_schema_registry_base import ( + KafkaSchemaRegistryBase, +) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, StaleEntityRemovalSourceReport, @@ -143,7 +146,7 @@ class KafkaSourceConfig( def get_kafka_consumer( connection: KafkaConsumerConnectionConfig, ) -> confluent_kafka.Consumer: - return confluent_kafka.Consumer( + consumer = confluent_kafka.Consumer( { "group.id": "test", "bootstrap.servers": connection.bootstrap, @@ -151,6 +154,13 @@ def get_kafka_consumer( } ) + if CallableConsumerConfig.is_callable_config(connection.consumer_config): + # As per documentation, we need to explicitly call the poll method to make sure OAuth callback gets executed + # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration + consumer.poll(timeout=30) + + return consumer + @dataclass class KafkaSourceReport(StaleEntityRemovalSourceReport): diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py similarity index 100% rename from metadata-ingestion/src/datahub/ingestion/source/kafka_connect.py rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_schema_registry_base.py similarity index 100% rename from metadata-ingestion/src/datahub/ingestion/source/kafka_schema_registry_base.py rename to metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_schema_registry_base.py diff --git a/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml b/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml new file mode 100644 index 00000000000000..34cf8fd47e658c --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/kafka_to_file_oauth.yml @@ -0,0 +1,20 @@ +run_id: kafka-test + +source: + type: kafka + config: + connection: + bootstrap: "localhost:29092" + schema_registry_url: "http://localhost:28081" + consumer_config: + security.protocol: "SASL_PLAINTEXT" + sasl.mechanism: "OAUTHBEARER" + oauth_cb: "oauth:create_token" + domain: + "urn:li:domain:sales": + allow: + - "key_value_topic" +sink: + type: file + config: + filename: "./kafka_mces.json" diff --git a/metadata-ingestion/tests/integration/kafka/oauth.py b/metadata-ingestion/tests/integration/kafka/oauth.py new file mode 100644 index 00000000000000..28cfee521d6c0f --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/oauth.py @@ -0,0 +1,14 @@ +import logging +from typing import Any, Tuple + +logger = logging.getLogger(__name__) + +MESSAGE: str = "OAuth token `create_token` callback" + + +def create_token(*args: Any, **kwargs: Any) -> Tuple[str, int]: + logger.warning(MESSAGE) + return ( + "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456", + 3600, + ) diff --git a/metadata-ingestion/tests/integration/kafka/test_kafka.py b/metadata-ingestion/tests/integration/kafka/test_kafka.py index dfdbea5de5cbfd..597889c8440b7a 100644 --- a/metadata-ingestion/tests/integration/kafka/test_kafka.py +++ b/metadata-ingestion/tests/integration/kafka/test_kafka.py @@ -1,10 +1,14 @@ +import logging import subprocess import pytest +import yaml from freezegun import freeze_time from datahub.ingestion.api.source import SourceCapability -from datahub.ingestion.source.kafka import KafkaSource +from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.source.kafka.kafka import KafkaSource +from tests.integration.kafka import oauth # type: ignore from tests.test_helpers import mce_helpers, test_connection_helpers from tests.test_helpers.click_helpers import run_datahub_cmd from tests.test_helpers.docker_helpers import wait_for_port @@ -99,3 +103,36 @@ def test_kafka_test_connection(mock_kafka_service, config_dict, is_success): SourceCapability.SCHEMA_METADATA: "Failed to establish a new connection" }, ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_kafka_oauth_callback( + mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time +): + # Run the metadata ingestion pipeline. + config_file = (test_resources_dir / "kafka_to_file_oauth.yml").resolve() + + log_file = tmp_path / "kafka_oauth_message.log" + + file_handler = logging.FileHandler( + str(log_file) + ) # Add a file handler to later validate a test-case + logging.getLogger().addHandler(file_handler) + + recipe: dict = {} + with open(config_file) as fp: + recipe = yaml.safe_load(fp) + + pipeline = Pipeline.create(recipe) + + pipeline.run() + + is_found: bool = False + with open(log_file, "r") as file: + for line_number, line in enumerate(file, 1): + if oauth.MESSAGE in line: + is_found = True + break + + assert is_found diff --git a/metadata-ingestion/tests/unit/api/test_pipeline.py b/metadata-ingestion/tests/unit/api/test_pipeline.py index 432d8e11c1c0b4..fe3d3160b729a1 100644 --- a/metadata-ingestion/tests/unit/api/test_pipeline.py +++ b/metadata-ingestion/tests/unit/api/test_pipeline.py @@ -33,7 +33,9 @@ class TestPipeline: @patch("confluent_kafka.Consumer", autospec=True) - @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True) + @patch( + "datahub.ingestion.source.kafka.kafka.KafkaSource.get_workunits", autospec=True + ) @patch("datahub.ingestion.sink.console.ConsoleSink.close", autospec=True) @freeze_time(FROZEN_TIME) def test_configure(self, mock_sink, mock_source, mock_consumer): @@ -198,7 +200,9 @@ def test_configure_with_rest_sink_with_additional_props_initializes_graph( assert pipeline.ctx.graph.config.token == pipeline.config.sink.config["token"] @freeze_time(FROZEN_TIME) - @patch("datahub.ingestion.source.kafka.KafkaSource.get_workunits", autospec=True) + @patch( + "datahub.ingestion.source.kafka.kafka.KafkaSource.get_workunits", autospec=True + ) def test_configure_with_file_sink_does_not_init_graph(self, mock_source, tmp_path): pipeline = Pipeline.create( { diff --git a/metadata-ingestion/tests/unit/test_confluent_schema_registry.py b/metadata-ingestion/tests/unit/test_confluent_schema_registry.py index b047cd16c52a97..3500636f00eddf 100644 --- a/metadata-ingestion/tests/unit/test_confluent_schema_registry.py +++ b/metadata-ingestion/tests/unit/test_confluent_schema_registry.py @@ -8,7 +8,7 @@ ) from datahub.ingestion.source.confluent_schema_registry import ConfluentSchemaRegistry -from datahub.ingestion.source.kafka import KafkaSourceConfig, KafkaSourceReport +from datahub.ingestion.source.kafka.kafka import KafkaSourceConfig, KafkaSourceReport class ConfluentSchemaRegistryTest(unittest.TestCase): diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py index b4e37d288a3041..dfd32085b77055 100644 --- a/metadata-ingestion/tests/unit/test_kafka_source.py +++ b/metadata-ingestion/tests/unit/test_kafka_source.py @@ -23,7 +23,7 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.workunit import MetadataWorkUnit -from datahub.ingestion.source.kafka import KafkaSource, KafkaSourceConfig +from datahub.ingestion.source.kafka.kafka import KafkaSource, KafkaSourceConfig from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( BrowsePathsClass, @@ -38,11 +38,13 @@ @pytest.fixture def mock_admin_client(): - with patch("datahub.ingestion.source.kafka.AdminClient", autospec=True) as mock: + with patch( + "datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True + ) as mock: yield mock -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_configuration(mock_kafka): ctx = PipelineContext(run_id="test") kafka_source = KafkaSource( @@ -53,7 +55,7 @@ def test_kafka_source_configuration(mock_kafka): assert mock_kafka.call_count == 1 -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_wildcard_topic(mock_kafka, mock_admin_client): mock_kafka_instance = mock_kafka.return_value mock_cluster_metadata = MagicMock() @@ -74,7 +76,7 @@ def test_kafka_source_workunits_wildcard_topic(mock_kafka, mock_admin_client): assert len(workunits) == 4 -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_topic_pattern(mock_kafka, mock_admin_client): mock_kafka_instance = mock_kafka.return_value mock_cluster_metadata = MagicMock() @@ -108,7 +110,7 @@ def test_kafka_source_workunits_topic_pattern(mock_kafka, mock_admin_client): assert len(workunits) == 4 -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_client): PLATFORM_INSTANCE = "kafka_cluster" PLATFORM = "kafka" @@ -160,7 +162,7 @@ def test_kafka_source_workunits_with_platform_instance(mock_kafka, mock_admin_cl assert f"/prod/{PLATFORM}/{PLATFORM_INSTANCE}" in browse_path_aspects[0].paths -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_no_platform_instance(mock_kafka, mock_admin_client): PLATFORM = "kafka" TOPIC_NAME = "test" @@ -204,7 +206,7 @@ def test_kafka_source_workunits_no_platform_instance(mock_kafka, mock_admin_clie assert f"/prod/{PLATFORM}" in browse_path_aspects[0].paths -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_close(mock_kafka, mock_admin_client): mock_kafka_instance = mock_kafka.return_value ctx = PipelineContext(run_id="test") @@ -223,7 +225,7 @@ def test_close(mock_kafka, mock_admin_client): "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient", autospec=True, ) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_workunits_schema_registry_subject_name_strategies( mock_kafka_consumer, mock_schema_registry_client, mock_admin_client ): @@ -415,7 +417,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient", autospec=True, ) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_ignore_warnings_on_schema_type( mock_kafka_consumer, mock_schema_registry_client, @@ -483,8 +485,8 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: assert kafka_source.report.warnings -@patch("datahub.ingestion.source.kafka.AdminClient", autospec=True) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_succeeds_with_admin_client_init_error( mock_kafka, mock_kafka_admin_client ): @@ -513,8 +515,8 @@ def test_kafka_source_succeeds_with_admin_client_init_error( assert len(workunits) == 2 -@patch("datahub.ingestion.source.kafka.AdminClient", autospec=True) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.AdminClient", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_succeeds_with_describe_configs_error( mock_kafka, mock_kafka_admin_client ): @@ -550,7 +552,7 @@ def test_kafka_source_succeeds_with_describe_configs_error( "datahub.ingestion.source.confluent_schema_registry.SchemaRegistryClient", autospec=True, ) -@patch("datahub.ingestion.source.kafka.confluent_kafka.Consumer", autospec=True) +@patch("datahub.ingestion.source.kafka.kafka.confluent_kafka.Consumer", autospec=True) def test_kafka_source_topic_meta_mappings( mock_kafka_consumer, mock_schema_registry_client, mock_admin_client ): From c3f9a9206d505d49e70b1795476c4928c87f2a42 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 22 Nov 2024 20:32:24 +0530 Subject: [PATCH 051/174] feat(ingest/mssql): include stored procedure lineage (#11912) Co-authored-by: Harshal Sheth --- .../ingestion/source/sql/mssql/job_models.py | 1 + .../ingestion/source/sql/mssql/source.py | 151 +- .../sql/mssql/stored_procedure_lineage.py | 84 + .../ingestion/source/sql/sql_common.py | 12 +- .../src/datahub/sql_parsing/datajob.py | 50 + .../src/datahub/sql_parsing/query_types.py | 11 +- .../datahub/sql_parsing/split_statements.py | 163 + .../sql_parsing/sql_parsing_aggregator.py | 1 - .../integration/sql_server/docker-compose.yml | 2 +- .../golden_mces_mssql_no_db_to_file.json | 396 ++- .../golden_mces_mssql_no_db_with_filter.json | 178 +- .../golden_mces_mssql_to_file.json | 178 +- ...golden_mces_mssql_with_lower_case_urn.json | 2649 ++++++++++++++++- .../procedures/DemoData.Foo.NewProc.json | 57 + .../procedures/demodata.foo.proc2.json | 57 + .../procedures/DemoData.Foo.NewProc.sql | 37 + .../procedures/demodata.foo.proc2.sql | 36 + .../integration/sql_server/setup/setup.sql | 53 +- .../mssql_with_lower_case_urn.yml | 1 - .../integration/sql_server/test_sql_server.py | 54 + .../unit/sql_parsing/test_split_statements.py | 51 + 21 files changed, 4074 insertions(+), 148 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py create mode 100644 metadata-ingestion/src/datahub/sql_parsing/datajob.py create mode 100644 metadata-ingestion/src/datahub/sql_parsing/split_statements.py create mode 100644 metadata-ingestion/tests/integration/sql_server/golden_files/procedures/DemoData.Foo.NewProc.json create mode 100644 metadata-ingestion/tests/integration/sql_server/golden_files/procedures/demodata.foo.proc2.json create mode 100644 metadata-ingestion/tests/integration/sql_server/procedures/DemoData.Foo.NewProc.sql create mode 100644 metadata-ingestion/tests/integration/sql_server/procedures/demodata.foo.proc2.sql create mode 100644 metadata-ingestion/tests/unit/sql_parsing/test_split_statements.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py index 21e7fad3343314..5107a4e38f64de 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/job_models.py @@ -101,6 +101,7 @@ class StoredProcedure: flow: Union[MSSQLJob, MSSQLProceduresContainer] type: str = "STORED_PROCEDURE" source: str = "mssql" + code: Optional[str] = None @property def full_type(self) -> str: diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py index c19b22a8622ca2..9ab9c76c30417f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py @@ -24,6 +24,8 @@ platform_name, support_status, ) +from datahub.ingestion.api.source import StructuredLogLevel +from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.sql.mssql.job_models import ( JobStep, @@ -36,6 +38,9 @@ ProcedureParameter, StoredProcedure, ) +from datahub.ingestion.source.sql.mssql.stored_procedure_lineage import ( + generate_procedure_lineage, +) from datahub.ingestion.source.sql.sql_common import ( SQLAlchemySource, SqlWorkUnit, @@ -51,6 +56,7 @@ StringTypeClass, UnionTypeClass, ) +from datahub.utilities.file_backed_collections import FileBackedList logger: logging.Logger = logging.getLogger(__name__) @@ -99,6 +105,10 @@ class SQLServerConfig(BasicSQLAlchemyConfig): default=False, description="Enable to convert the SQL Server assets urns to lowercase", ) + include_lineage: bool = Field( + default=True, + description="Enable lineage extraction for stored procedures", + ) @pydantic.validator("uri_args") def passwords_match(cls, v, values, **kwargs): @@ -161,6 +171,7 @@ def __init__(self, config: SQLServerConfig, ctx: PipelineContext): self.current_database = None self.table_descriptions: Dict[str, str] = {} self.column_descriptions: Dict[str, str] = {} + self.stored_procedures: FileBackedList[StoredProcedure] = FileBackedList() if self.config.include_descriptions: for inspector in self.get_inspectors(): db_name: str = self.get_db_name(inspector) @@ -374,7 +385,7 @@ def loop_jobs( def loop_job_steps( self, job: MSSQLJob, job_steps: Dict[str, Any] ) -> Iterable[MetadataWorkUnit]: - for step_id, step_data in job_steps.items(): + for _step_id, step_data in job_steps.items(): step = JobStep( job_name=job.formatted_name, step_name=step_data["step_name"], @@ -412,37 +423,44 @@ def loop_stored_procedures( # noqa: C901 if procedures: yield from self.construct_flow_workunits(data_flow=data_flow) for procedure in procedures: - upstream = self._get_procedure_upstream(conn, procedure) - downstream = self._get_procedure_downstream(conn, procedure) - data_job = MSSQLDataJob( - entity=procedure, - ) - # TODO: because of this upstream and downstream are more dependencies, - # can't be used as DataJobInputOutput. - # Should be reorganized into lineage. - data_job.add_property("procedure_depends_on", str(upstream.as_property)) - data_job.add_property( - "depending_on_procedure", str(downstream.as_property) - ) - procedure_definition, procedure_code = self._get_procedure_code( - conn, procedure - ) - if procedure_definition: - data_job.add_property("definition", procedure_definition) - if sql_config.include_stored_procedures_code and procedure_code: - data_job.add_property("code", procedure_code) - procedure_inputs = self._get_procedure_inputs(conn, procedure) - properties = self._get_procedure_properties(conn, procedure) - data_job.add_property( - "input parameters", str([param.name for param in procedure_inputs]) - ) - for param in procedure_inputs: - data_job.add_property( - f"parameter {param.name}", str(param.properties) - ) - for property_name, property_value in properties.items(): - data_job.add_property(property_name, str(property_value)) - yield from self.construct_job_workunits(data_job) + yield from self._process_stored_procedure(conn, procedure) + + def _process_stored_procedure( + self, conn: Connection, procedure: StoredProcedure + ) -> Iterable[MetadataWorkUnit]: + upstream = self._get_procedure_upstream(conn, procedure) + downstream = self._get_procedure_downstream(conn, procedure) + data_job = MSSQLDataJob( + entity=procedure, + ) + # TODO: because of this upstream and downstream are more dependencies, + # can't be used as DataJobInputOutput. + # Should be reorganized into lineage. + data_job.add_property("procedure_depends_on", str(upstream.as_property)) + data_job.add_property("depending_on_procedure", str(downstream.as_property)) + procedure_definition, procedure_code = self._get_procedure_code(conn, procedure) + procedure.code = procedure_code + if procedure_definition: + data_job.add_property("definition", procedure_definition) + if procedure_code and self.config.include_stored_procedures_code: + data_job.add_property("code", procedure_code) + procedure_inputs = self._get_procedure_inputs(conn, procedure) + properties = self._get_procedure_properties(conn, procedure) + data_job.add_property( + "input parameters", str([param.name for param in procedure_inputs]) + ) + for param in procedure_inputs: + data_job.add_property(f"parameter {param.name}", str(param.properties)) + for property_name, property_value in properties.items(): + data_job.add_property(property_name, str(property_value)) + if self.config.include_lineage: + # These will be used to construct lineage + self.stored_procedures.append(procedure) + yield from self.construct_job_workunits( + data_job, + # For stored procedure lineage is ingested later + include_lineage=False, + ) @staticmethod def _get_procedure_downstream( @@ -546,8 +564,8 @@ def _get_procedure_code( code_list.append(row["Text"]) if code_slice_text in re.sub(" +", " ", row["Text"].lower()).strip(): code_slice_index = index - definition = "\n".join(code_list[:code_slice_index]) - code = "\n".join(code_list[code_slice_index:]) + definition = "".join(code_list[:code_slice_index]) + code = "".join(code_list[code_slice_index:]) except ResourceClosedError: logger.warning( "Connection was closed from procedure '%s'", @@ -602,16 +620,18 @@ def _get_stored_procedures( def construct_job_workunits( self, data_job: MSSQLDataJob, + include_lineage: bool = True, ) -> Iterable[MetadataWorkUnit]: yield MetadataChangeProposalWrapper( entityUrn=data_job.urn, aspect=data_job.as_datajob_info_aspect, ).as_workunit() - yield MetadataChangeProposalWrapper( - entityUrn=data_job.urn, - aspect=data_job.as_datajob_input_output_aspect, - ).as_workunit() + if include_lineage: + yield MetadataChangeProposalWrapper( + entityUrn=data_job.urn, + aspect=data_job.as_datajob_input_output_aspect, + ).as_workunit() # TODO: Add SubType when it appear def construct_flow_workunits( @@ -664,3 +684,58 @@ def get_identifier( if self.config.convert_urns_to_lowercase else qualified_table_name ) + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + yield from super().get_workunits_internal() + + # This is done at the end so that we will have access to tables + # from all databases in schema_resolver and discovered_tables + for procedure in self.stored_procedures: + with self.report.report_exc( + message="Failed to parse stored procedure lineage", + context=procedure.full_name, + level=StructuredLogLevel.WARN, + ): + yield from auto_workunit( + generate_procedure_lineage( + schema_resolver=self.schema_resolver, + procedure=procedure, + procedure_job_urn=MSSQLDataJob(entity=procedure).urn, + is_temp_table=self.is_temp_table, + ) + ) + + def is_temp_table(self, name: str) -> bool: + try: + parts = name.split(".") + table_name = parts[-1] + schema_name = parts[-2] + db_name = parts[-3] + + if table_name.startswith("#"): + return True + + # This is also a temp table if + # 1. this name would be allowed by the dataset patterns, and + # 2. we have a list of discovered tables, and + # 3. it's not in the discovered tables list + if ( + self.config.database_pattern.allowed(db_name) + and self.config.schema_pattern.allowed(schema_name) + and self.config.table_pattern.allowed(name) + and self.standardize_identifier_case(name) + not in self.discovered_datasets + ): + logger.debug(f"inferred as temp table {name}") + return True + + except Exception: + logger.warning(f"Error parsing table name {name} ") + return False + + def standardize_identifier_case(self, table_ref_str: str) -> str: + return ( + table_ref_str.lower() + if self.config.convert_urns_to_lowercase + else table_ref_str + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py new file mode 100644 index 00000000000000..b979a270a55282 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/stored_procedure_lineage.py @@ -0,0 +1,84 @@ +import logging +from typing import Callable, Iterable, Optional + +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.source.sql.mssql.job_models import StoredProcedure +from datahub.metadata.schema_classes import DataJobInputOutputClass +from datahub.sql_parsing.datajob import to_datajob_input_output +from datahub.sql_parsing.schema_resolver import SchemaResolver +from datahub.sql_parsing.split_statements import split_statements +from datahub.sql_parsing.sql_parsing_aggregator import ( + ObservedQuery, + SqlParsingAggregator, +) + +logger = logging.getLogger(__name__) + + +def parse_procedure_code( + *, + schema_resolver: SchemaResolver, + default_db: Optional[str], + default_schema: Optional[str], + code: str, + is_temp_table: Callable[[str], bool], + raise_: bool = False, +) -> Optional[DataJobInputOutputClass]: + aggregator = SqlParsingAggregator( + platform=schema_resolver.platform, + env=schema_resolver.env, + schema_resolver=schema_resolver, + generate_lineage=True, + generate_queries=False, + generate_usage_statistics=False, + generate_operations=False, + generate_query_subject_fields=False, + generate_query_usage_statistics=False, + is_temp_table=is_temp_table, + ) + for query in split_statements(code): + # TODO: We should take into account `USE x` statements. + aggregator.add_observed_query( + observed=ObservedQuery( + default_db=default_db, + default_schema=default_schema, + query=query, + ) + ) + if aggregator.report.num_observed_queries_failed and raise_: + logger.info(aggregator.report.as_string()) + raise ValueError( + f"Failed to parse {aggregator.report.num_observed_queries_failed} queries." + ) + + mcps = list(aggregator.gen_metadata()) + return to_datajob_input_output( + mcps=mcps, + ignore_extra_mcps=True, + ) + + +# Is procedure handling generic enough to be added to SqlParsingAggregator? +def generate_procedure_lineage( + *, + schema_resolver: SchemaResolver, + procedure: StoredProcedure, + procedure_job_urn: str, + is_temp_table: Callable[[str], bool] = lambda _: False, + raise_: bool = False, +) -> Iterable[MetadataChangeProposalWrapper]: + if procedure.code: + datajob_input_output = parse_procedure_code( + schema_resolver=schema_resolver, + default_db=procedure.db, + default_schema=procedure.schema, + code=procedure.code, + is_temp_table=is_temp_table, + raise_=raise_, + ) + + if datajob_input_output: + yield MetadataChangeProposalWrapper( + entityUrn=procedure_job_urn, + aspect=datajob_input_output, + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index e5779791ed4120..ae6116326da33c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -392,6 +392,7 @@ def __init__(self, config: SQLCommonConfig, ctx: PipelineContext, platform: str) platform_instance=self.config.platform_instance, env=self.config.env, ) + self.discovered_datasets: Set[str] = set() self._view_definition_cache: MutableMapping[str, str] if self.config.use_file_backed_cache: self._view_definition_cache = FileBackedDict[str]() @@ -831,8 +832,9 @@ def _process_table( self._classify(dataset_name, schema, table, data_reader, schema_metadata) dataset_snapshot.aspects.append(schema_metadata) - if self.config.include_view_lineage: + if self._save_schema_to_resolver(): self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata) + self.discovered_datasets.add(dataset_name) db_name = self.get_db_name(inspector) yield from self.add_table_to_schema_container( @@ -1126,8 +1128,9 @@ def _process_view( columns, canonical_schema=schema_fields, ) - if self.config.include_view_lineage: + if self._save_schema_to_resolver(): self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata) + self.discovered_datasets.add(dataset_name) description, properties, _ = self.get_table_properties(inspector, schema, view) try: view_definition = inspector.get_view_definition(view, schema) @@ -1190,6 +1193,11 @@ def _process_view( domain_registry=self.domain_registry, ) + def _save_schema_to_resolver(self): + return self.config.include_view_lineage or ( + hasattr(self.config, "include_lineage") and self.config.include_lineage + ) + def _run_sql_parser( self, view_identifier: str, query: str, schema_resolver: SchemaResolver ) -> Optional[SqlParsingResult]: diff --git a/metadata-ingestion/src/datahub/sql_parsing/datajob.py b/metadata-ingestion/src/datahub/sql_parsing/datajob.py new file mode 100644 index 00000000000000..215b207c3dcf51 --- /dev/null +++ b/metadata-ingestion/src/datahub/sql_parsing/datajob.py @@ -0,0 +1,50 @@ +import logging +from typing import Iterable, List, Optional + +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.metadata.schema_classes import ( + DataJobInputOutputClass, + FineGrainedLineageClass, + UpstreamLineageClass, +) + +logger = logging.getLogger(__name__) + + +def to_datajob_input_output( + *, mcps: Iterable[MetadataChangeProposalWrapper], ignore_extra_mcps: bool = True +) -> Optional[DataJobInputOutputClass]: + inputDatasets: List[str] = [] + outputDatasets: List[str] = [] + fineGrainedLineages: List[FineGrainedLineageClass] = [] + for mcp in mcps: + # TODO: Represent simple write operations without lineage as outputDatasets. + + upstream_lineage = mcp.as_workunit().get_aspect_of_type(UpstreamLineageClass) + if upstream_lineage is not None: + if mcp.entityUrn and mcp.entityUrn not in outputDatasets: + outputDatasets.append(mcp.entityUrn) + + for upstream in upstream_lineage.upstreams: + if upstream.dataset not in inputDatasets: + inputDatasets.append(upstream.dataset) + + if upstream_lineage.fineGrainedLineages: + for fineGrainedLineage in upstream_lineage.fineGrainedLineages: + fineGrainedLineages.append(fineGrainedLineage) + + elif ignore_extra_mcps: + pass + else: + raise ValueError( + f"Expected an upstreamLineage aspect, got {mcp.aspectName} for {mcp.entityUrn}" + ) + + if not inputDatasets and not outputDatasets: + return None + + return DataJobInputOutputClass( + inputDatasets=inputDatasets, + outputDatasets=outputDatasets, + fineGrainedLineages=fineGrainedLineages, + ) diff --git a/metadata-ingestion/src/datahub/sql_parsing/query_types.py b/metadata-ingestion/src/datahub/sql_parsing/query_types.py index 2acad19418c113..802fb3e993f428 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/query_types.py +++ b/metadata-ingestion/src/datahub/sql_parsing/query_types.py @@ -14,7 +14,16 @@ def _is_temp_table(table: sqlglot.exp.Table, dialect: sqlglot.Dialect) -> bool: identifier: sqlglot.exp.Identifier = table.this return identifier.args.get("temporary") or ( - is_dialect_instance(dialect, "redshift") and identifier.name.startswith("#") + # These dialects use # as a prefix for temp tables. + is_dialect_instance( + dialect, + [ + "redshift", + "mssql", + # sybase is another one, but we don't support that dialect yet. + ], + ) + and identifier.name.startswith("#") ) diff --git a/metadata-ingestion/src/datahub/sql_parsing/split_statements.py b/metadata-ingestion/src/datahub/sql_parsing/split_statements.py new file mode 100644 index 00000000000000..42dda4e62158b0 --- /dev/null +++ b/metadata-ingestion/src/datahub/sql_parsing/split_statements.py @@ -0,0 +1,163 @@ +import re +from enum import Enum +from typing import Generator, List, Tuple + +CONTROL_FLOW_KEYWORDS = [ + "GO", + r"BEGIN\w+TRY", + r"BEGIN\w+CATCH", + "BEGIN", + r"END\w+TRY", + r"END\w+CATCH", + "END", +] + +# There's an exception to this rule, which is when the statement +# is preceeded by a CTE. +FORCE_NEW_STATEMENT_KEYWORDS = [ + # SELECT is used inside queries as well, so we can't include it here. + "INSERT", + "UPDATE", + "DELETE", + "MERGE", +] + + +class ParserState(Enum): + NORMAL = 1 + STRING = 2 + COMMENT = 3 + MULTILINE_COMMENT = 4 + + +def _is_keyword_at_position(sql: str, pos: int, keyword: str) -> bool: + """ + Check if a keyword exists at the given position using regex word boundaries. + """ + if pos + len(keyword) > len(sql): + return False + + # If we're not at a word boundary, we can't generate a keyword. + if pos > 0 and not ( + bool(re.match(r"\w\W", sql[pos - 1 : pos + 1])) + or bool(re.match(r"\W\w", sql[pos - 1 : pos + 1])) + ): + return False + + pattern = rf"^{re.escape(keyword)}\b" + match = re.match(pattern, sql[pos:], re.IGNORECASE) + return bool(match) + + +def _look_ahead_for_keywords( + sql: str, pos: int, keywords: List[str] +) -> Tuple[bool, str, int]: + """ + Look ahead for SQL keywords at the current position. + """ + + for keyword in keywords: + if _is_keyword_at_position(sql, pos, keyword): + return True, keyword, len(keyword) + return False, "", 0 + + +def split_statements(sql: str) -> Generator[str, None, None]: + """ + Split T-SQL code into individual statements, handling various SQL constructs. + """ + if not sql or not sql.strip(): + return + + current_statement: List[str] = [] + state = ParserState.NORMAL + i = 0 + + def yield_if_complete() -> Generator[str, None, None]: + statement = "".join(current_statement).strip() + if statement: + yield statement + current_statement.clear() + + prev_real_char = "\0" # the most recent non-whitespace, non-comment character + while i < len(sql): + c = sql[i] + next_char = sql[i + 1] if i < len(sql) - 1 else "\0" + + if state == ParserState.NORMAL: + if c == "'": + state = ParserState.STRING + current_statement.append(c) + prev_real_char = c + elif c == "-" and next_char == "-": + state = ParserState.COMMENT + current_statement.append(c) + current_statement.append(next_char) + i += 1 + elif c == "/" and next_char == "*": + state = ParserState.MULTILINE_COMMENT + current_statement.append(c) + current_statement.append(next_char) + i += 1 + else: + most_recent_real_char = prev_real_char + if not c.isspace(): + prev_real_char = c + + is_control_keyword, keyword, keyword_len = _look_ahead_for_keywords( + sql, i, keywords=CONTROL_FLOW_KEYWORDS + ) + if is_control_keyword: + # Yield current statement if any + yield from yield_if_complete() + # Yield keyword as its own statement + yield keyword + i += keyword_len + continue + + ( + is_force_new_statement_keyword, + keyword, + keyword_len, + ) = _look_ahead_for_keywords( + sql, i, keywords=FORCE_NEW_STATEMENT_KEYWORDS + ) + if ( + is_force_new_statement_keyword and most_recent_real_char != ")" + ): # usually we'd have a close paren that closes a CTE + # Force termination of current statement + yield from yield_if_complete() + + current_statement.append(keyword) + i += keyword_len + continue + + elif c == ";": + yield from yield_if_complete() + else: + current_statement.append(c) + + elif state == ParserState.STRING: + current_statement.append(c) + if c == "'" and next_char == "'": + current_statement.append(next_char) + i += 1 + elif c == "'": + state = ParserState.NORMAL + + elif state == ParserState.COMMENT: + current_statement.append(c) + if c == "\n": + state = ParserState.NORMAL + + elif state == ParserState.MULTILINE_COMMENT: + current_statement.append(c) + if c == "*" and next_char == "/": + current_statement.append(next_char) + i += 1 + state = ParserState.NORMAL + + i += 1 + + # Handle the last statement + yield from yield_if_complete() diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 360ccd7bf35073..44f0d7be7aad9a 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -762,7 +762,6 @@ def add_observed_query( This assumes that queries come in order of increasing timestamps. """ - self.report.num_observed_queries += 1 # All queries with no session ID are assumed to be part of the same session. diff --git a/metadata-ingestion/tests/integration/sql_server/docker-compose.yml b/metadata-ingestion/tests/integration/sql_server/docker-compose.yml index 1046321e4f7205..aed70503903c03 100644 --- a/metadata-ingestion/tests/integration/sql_server/docker-compose.yml +++ b/metadata-ingestion/tests/integration/sql_server/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: testsqlserver: - image: "mcr.microsoft.com/mssql/server:latest" + image: "mcr.microsoft.com/mssql/server:2022-latest" platform: linux/amd64 container_name: "testsqlserver" environment: diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 4302c41140dc6c..54821347fd28b8 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "01afcab8-187c-459f-828e-727196a1832d", + "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-21 21:01:26.550000", - "date_modified": "2024-11-21 21:01:26.690000", + "date_created": "2024-11-22 12:58:03.260000", + "date_modified": "2024-11-22 12:58:03.440000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1496,6 +1496,138 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "age_dist", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.age_dist", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", @@ -2150,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-21 21:01:26.483000", - "date_modified": "2024-11-21 21:01:26.483000" + "date_created": "2024-11-22 12:58:03.137000", + "date_modified": "2024-11-22 12:58:03.137000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2168,14 +2300,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", + "aspectName": "dataJobInfo", "aspect": { "json": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [] + "customProperties": { + "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}", + "depending_on_procedure": "{}", + "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", + "input parameters": "[]", + "date_created": "2024-11-22 12:58:03.140000", + "date_modified": "2024-11-22 12:58:03.140000" + }, + "externalUrl": "", + "name": "DemoData.Foo.NewProc", + "type": { + "string": "MSSQL_STORED_PROCEDURE" + } } }, "systemMetadata": { @@ -4256,6 +4398,159 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n", + "is_view": "True" + }, + "name": "View1", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "NewData.FooNew.View1", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + }, + { + "id": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", @@ -4611,6 +4906,55 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),firstname)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),firstname)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),lastname)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),lastname)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", @@ -4643,6 +4987,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", @@ -4690,5 +5050,21 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 0a50556edc6388..3836e587ef8cf4 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "01afcab8-187c-459f-828e-727196a1832d", + "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-21 21:01:26.550000", - "date_modified": "2024-11-21 21:01:26.690000", + "date_created": "2024-11-22 12:58:03.260000", + "date_modified": "2024-11-22 12:58:03.440000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1496,6 +1496,138 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "age_dist", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.age_dist", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", @@ -2150,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-21 21:01:26.483000", - "date_modified": "2024-11-21 21:01:26.483000" + "date_created": "2024-11-22 12:58:03.137000", + "date_modified": "2024-11-22 12:58:03.137000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2168,14 +2300,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", + "aspectName": "dataJobInfo", "aspect": { "json": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [] + "customProperties": { + "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}", + "depending_on_procedure": "{}", + "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", + "input parameters": "[]", + "date_created": "2024-11-22 12:58:03.140000", + "date_modified": "2024-11-22 12:58:03.140000" + }, + "externalUrl": "", + "name": "DemoData.Foo.NewProc", + "type": { + "string": "MSSQL_STORED_PROCEDURE" + } } }, "systemMetadata": { @@ -2571,6 +2713,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 0a50556edc6388..3836e587ef8cf4 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "01afcab8-187c-459f-828e-727196a1832d", + "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-21 21:01:26.550000", - "date_modified": "2024-11-21 21:01:26.690000", + "date_created": "2024-11-22 12:58:03.260000", + "date_modified": "2024-11-22 12:58:03.440000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1496,6 +1496,138 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "age_dist", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.age_dist", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", @@ -2150,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-21 21:01:26.483000", - "date_modified": "2024-11-21 21:01:26.483000" + "date_created": "2024-11-22 12:58:03.137000", + "date_modified": "2024-11-22 12:58:03.137000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2168,14 +2300,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", + "aspectName": "dataJobInfo", "aspect": { "json": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [] + "customProperties": { + "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}", + "depending_on_procedure": "{}", + "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", + "input parameters": "[]", + "date_created": "2024-11-22 12:58:03.140000", + "date_modified": "2024-11-22 12:58:03.140000" + }, + "externalUrl": "", + "name": "DemoData.Foo.NewProc", + "type": { + "string": "MSSQL_STORED_PROCEDURE" + } } }, "systemMetadata": { @@ -2571,6 +2713,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index 0279a94084ce56..ebcadcc11dcbfa 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "01afcab8-187c-459f-828e-727196a1832d", + "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-21 21:01:26.550000", - "date_modified": "2024-11-21 21:01:26.690000", + "date_created": "2024-11-22 12:58:03.260000", + "date_modified": "2024-11-22 12:58:03.440000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1496,6 +1496,138 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "age_dist", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "demodata.foo.age_dist", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Count", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b275b7c099ce32f3faf1817cb054b100", + "urn": "urn:li:container:b275b7c099ce32f3faf1817cb054b100" + }, + { + "id": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9", + "urn": "urn:li:container:046d11ae7c0bc9bde45993041ac011c9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", @@ -2150,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-21 21:01:26.483000", - "date_modified": "2024-11-21 21:01:26.483000" + "date_created": "2024-11-22 12:58:03.137000", + "date_modified": "2024-11-22 12:58:03.137000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2168,14 +2300,24 @@ }, { "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", "changeType": "UPSERT", - "aspectName": "dataJobInputOutput", + "aspectName": "dataJobInfo", "aspect": { "json": { - "inputDatasets": [], - "outputDatasets": [], - "inputDatajobs": [] + "customProperties": { + "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}", + "depending_on_procedure": "{}", + "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", + "input parameters": "[]", + "date_created": "2024-11-22 12:58:03.140000", + "date_modified": "2024-11-22 12:58:03.140000" + }, + "externalUrl": "", + "name": "DemoData.Foo.NewProc", + "type": { + "string": "MSSQL_STORED_PROCEDURE" + } } }, "systemMetadata": { @@ -2515,68 +2657,19 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "containerProperties", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", - "type": "VIEW" - } - ], - "fineGrainedLineages": [ - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" - ], - "confidenceScore": 1.0 - } - ] + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData" + }, + "name": "NewData", + "env": "PROD" } }, "systemMetadata": { @@ -2586,8 +2679,8 @@ } }, { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2602,8 +2695,2418 @@ } }, { - "entityType": "dataFlow", - "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_accessadmin" + }, + "name": "db_accessadmin", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:8b7691fec458d7383d5bc4e213831375", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_backupoperator" + }, + "name": "db_backupoperator", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:523d13eddd725607ec835a2459b05c9c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_datareader" + }, + "name": "db_datareader", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:29bd421b2225a415df9c750e77404c66", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_datawriter" + }, + "name": "db_datawriter", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a3c02df4bcc7280a89f539b793b04197", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_ddladmin" + }, + "name": "db_ddladmin", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c3b5d1cdc69a7d8faf0e1981e89b89d1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_denydatareader" + }, + "name": "db_denydatareader", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b937d85ae7545dc769766008a332f42", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_denydatawriter" + }, + "name": "db_denydatawriter", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a399d8bb765028abb9e55ae39846ca5e", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_owner" + }, + "name": "db_owner", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:457efe38f0aec2af9ad681cf1b43b1cb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "db_securityadmin" + }, + "name": "db_securityadmin", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1d87783ffe7e82210365dff4ca8ee7d1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "dbo" + }, + "name": "dbo", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:269d0067d130eda0399a534fc787054c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:269d0067d130eda0399a534fc787054c" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "ProductsNew", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "newdata.dbo.productsnew", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ProductName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "MONEY", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.dbo.productsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + }, + { + "id": "urn:li:container:269d0067d130eda0399a534fc787054c", + "urn": "urn:li:container:269d0067d130eda0399a534fc787054c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "FooNew" + }, + "name": "FooNew", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "ItemsNew", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "newdata.foonew.itemsnew", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ItemName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "SMALLMONEY", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.itemsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + }, + { + "id": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "PersonsNew", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "newdata.foonew.personsnew", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + }, + { + "id": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "view_definition": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n", + "is_view": "True" + }, + "name": "View1", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "newdata.foonew.view1", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "LastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "FirstName", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "CREATE VIEW FooNew.View1 AS\nSELECT LastName, FirstName\nFROM FooNew.PersonsNew\nWHERE Age > 18\n", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + }, + { + "id": "urn:li:container:f721da08adde46586c0f113287cb60d1", + "urn": "urn:li:container:f721da08adde46586c0f113287cb60d1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "guest" + }, + "name": "guest", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f3cb304e29e178d0615ed5ee6aa4ad58", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "INFORMATION_SCHEMA" + }, + "name": "INFORMATION_SCHEMA", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:752bb2abafeb2dae8f4adc7ffd547780", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "mssql", + "env": "PROD", + "database": "NewData", + "schema": "sys" + }, + "name": "sys", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:mssql" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Schema" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:46b713e3c7754c51649899f0f284ce34", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:0a12bec9e9271b0db039923a770d75e5", + "urn": "urn:li:container:0a12bec9e9271b0db039923a770d75e5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),Age)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),FirstName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),FirstName)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),LastName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD),LastName)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)", + "type": "VIEW" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),FirstName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),FirstName)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),LastName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),LastName)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD),Age)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),tempid)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD),Age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),Age)" + ], + "confidenceScore": 0.35 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(mssql,Weekly Demo Data Backup,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", "changeType": "UPSERT", "aspectName": "status", "aspect": { diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/DemoData.Foo.NewProc.json b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/DemoData.Foo.NewProc.json new file mode 100644 index 00000000000000..609e3a6f429452 --- /dev/null +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/DemoData.Foo.NewProc.json @@ -0,0 +1,57 @@ +[ +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD),age)" + ], + "confidenceScore": 0.2 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),tempid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),tempid)" + ], + "confidenceScore": 0.2 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),name)" + ], + "confidenceScore": 0.2 + } + ] + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/demodata.foo.proc2.json b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/demodata.foo.proc2.json new file mode 100644 index 00000000000000..8ebd1c065ebf94 --- /dev/null +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/procedures/demodata.foo.proc2.json @@ -0,0 +1,57 @@ +[ +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.foo.stored_procedures,PROD),proc2)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.personsnew,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.age_dist,PROD),age)" + ], + "confidenceScore": 0.2 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),tempid)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),tempid)" + ], + "confidenceScore": 0.2 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD),name)" + ], + "confidenceScore": 0.2 + } + ] + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/procedures/DemoData.Foo.NewProc.sql b/metadata-ingestion/tests/integration/sql_server/procedures/DemoData.Foo.NewProc.sql new file mode 100644 index 00000000000000..52a8d1327653b2 --- /dev/null +++ b/metadata-ingestion/tests/integration/sql_server/procedures/DemoData.Foo.NewProc.sql @@ -0,0 +1,37 @@ +CREATE PROCEDURE [Foo].[NewProc] + AS + BEGIN + --insert into items table from salesreason table + insert into Foo.Items (ID, ItemName) + SELECT TempID, Name + FROM Foo.SalesReason; + + + IF OBJECT_ID('Foo.age_dist', 'U') IS NULL + + BEGIN + -- Create and populate if table doesn't exist + SELECT Age, COUNT(*) as Count + INTO Foo.age_dist + FROM Foo.Persons + GROUP BY Age + END + ELSE + BEGIN + -- Update existing table + TRUNCATE TABLE Foo.age_dist; + + INSERT INTO Foo.age_dist (Age, Count) + SELECT Age, COUNT(*) as Count + FROM Foo.Persons + GROUP BY Age + END + + SELECT * INTO #TempTable FROM NewData.FooNew.PersonsNew + + UPDATE DemoData.Foo.Persons + SET Age = t.Age + FROM DemoData.Foo.Persons p + JOIN #TempTable t ON p.ID = t.ID + + END \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/procedures/demodata.foo.proc2.sql b/metadata-ingestion/tests/integration/sql_server/procedures/demodata.foo.proc2.sql new file mode 100644 index 00000000000000..69194a8d2c5464 --- /dev/null +++ b/metadata-ingestion/tests/integration/sql_server/procedures/demodata.foo.proc2.sql @@ -0,0 +1,36 @@ +CREATE PROCEDURE [foo].[proc2] + AS + BEGIN + --insert into items table from salesreason table + insert into foo.items (id, itemame) + SELECT tempid, name + FROM foo.salesreason; + + + IF OBJECT_ID('foo.age_dist', 'U') IS NULL + + BEGIN + -- Create and populate if table doesn't exist + SELECT age, COUNT(*) as count + INTO foo.age_dist + FROM foo.persons + GROUP BY age + END + ELSE + BEGIN + -- Update existing table + TRUNCATE TABLE foo.age_dist; + + INSERT INTO foo.age_dist (age, count) + SELECT age, COUNT(*) as count + FROM foo.persons + GROUP BY age + END + + SELECT * INTO #temptable FROM newdata.foonew.personsnew + + UPDATE demodata.foo.persons + SET age = t.age + FROM demodata.foo.persons p + JOIN #temptable t ON p.ID = t.ID + END \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql index f495db3b91cfae..0c3c7ee2fd29e3 100644 --- a/metadata-ingestion/tests/integration/sql_server/setup/setup.sql +++ b/metadata-ingestion/tests/integration/sql_server/setup/setup.sql @@ -1,3 +1,4 @@ +DROP DATABASE IF EXISTS NewData; CREATE DATABASE NewData; GO USE NewData; @@ -14,7 +15,14 @@ CREATE TABLE FooNew.PersonsNew ( FirstName varchar(255), Age int ); +GO +CREATE VIEW FooNew.View1 AS +SELECT LastName, FirstName +FROM FooNew.PersonsNew +WHERE Age > 18 +GO +DROP DATABASE IF EXISTS DemoData; CREATE DATABASE DemoData; GO USE DemoData; @@ -47,11 +55,54 @@ CREATE TABLE Foo.SalesReason ) ; GO +DROP PROCEDURE IF EXISTS [Foo].[Proc.With.SpecialChar]; +GO CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT AS SELECT @ID AS ThatDB; GO +DROP PROCEDURE IF EXISTS [Foo].[NewProc]; +GO +CREATE PROCEDURE [Foo].[NewProc] + AS + BEGIN + --insert into items table from salesreason table + insert into Foo.Items (ID, ItemName) + SELECT TempID, Name + FROM Foo.SalesReason; + + + IF OBJECT_ID('Foo.age_dist', 'U') IS NULL + BEGIN + -- Create and populate if table doesn't exist + SELECT Age, COUNT(*) as Count + INTO Foo.age_dist + FROM Foo.Persons + GROUP BY Age + END + ELSE + BEGIN + -- Update existing table + TRUNCATE TABLE Foo.age_dist; + + INSERT INTO Foo.age_dist (Age, Count) + SELECT Age, COUNT(*) as Count + FROM Foo.Persons + GROUP BY Age + END + + SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew + + UPDATE DemoData.Foo.Persons + SET Age = t.Age + FROM DemoData.Foo.Persons p + JOIN #TEMPTABLE t ON p.ID = t.ID + + END +GO + +EXEC Foo.NewProc GO EXEC sys.sp_addextendedproperty @name = N'MS_Description', @@ -93,4 +144,4 @@ EXEC sp_attach_schedule GO EXEC dbo.sp_add_jobserver @job_name = N'Weekly Demo Data Backup' -GO +GO \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml index ff1179034833f9..94128810f026b9 100644 --- a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml +++ b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_with_lower_case_urn.yml @@ -5,7 +5,6 @@ source: config: username: sa password: test!Password - database: DemoData host_port: localhost:21433 convert_urns_to_lowercase: true # use_odbc: True diff --git a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py index 1f418ffbd32ea9..b969f77b4c3c18 100644 --- a/metadata-ingestion/tests/integration/sql_server/test_sql_server.py +++ b/metadata-ingestion/tests/integration/sql_server/test_sql_server.py @@ -1,9 +1,16 @@ import os +import pathlib import subprocess import time +from pathlib import Path import pytest +from datahub.ingestion.source.sql.mssql.job_models import StoredProcedure +from datahub.ingestion.source.sql.mssql.stored_procedure_lineage import ( + generate_procedure_lineage, +) +from datahub.sql_parsing.schema_resolver import SchemaResolver from tests.test_helpers import mce_helpers from tests.test_helpers.click_helpers import run_datahub_cmd from tests.test_helpers.docker_helpers import cleanup_image, wait_for_port @@ -57,3 +64,50 @@ def test_mssql_ingest(mssql_runner, pytestconfig, tmp_path, mock_time, config_fi r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['date_modified'\]", ], ) + + +PROCEDURE_SQLS_DIR = pathlib.Path(__file__).parent / "procedures" +PROCEDURES_GOLDEN_DIR = pathlib.Path(__file__).parent / "golden_files/procedures/" +procedure_sqls = [sql_file.name for sql_file in PROCEDURE_SQLS_DIR.iterdir()] + + +@pytest.mark.parametrize("procedure_sql_file", procedure_sqls) +@pytest.mark.integration +def test_stored_procedure_lineage( + pytestconfig: pytest.Config, procedure_sql_file: str +) -> None: + sql_file_path = PROCEDURE_SQLS_DIR / procedure_sql_file + procedure_code = sql_file_path.read_text() + + # Procedure file is named as .. + splits = procedure_sql_file.split(".") + db = splits[0] + schema = splits[1] + name = splits[2] + + procedure = StoredProcedure( + db=db, + schema=schema, + name=name, + flow=None, # type: ignore # flow is not used in this test + code=procedure_code, + ) + data_job_urn = f"urn:li:dataJob:(urn:li:dataFlow:(mssql,{db}.{schema}.stored_procedures,PROD),{name})" + + schema_resolver = SchemaResolver(platform="mssql") + + mcps = list( + generate_procedure_lineage( + schema_resolver=schema_resolver, + procedure=procedure, + procedure_job_urn=data_job_urn, + is_temp_table=lambda name: "temp" in name.lower(), + ) + ) + mce_helpers.check_goldens_stream( + pytestconfig, + outputs=mcps, + golden_path=( + PROCEDURES_GOLDEN_DIR / Path(procedure_sql_file).with_suffix(".json") + ), + ) diff --git a/metadata-ingestion/tests/unit/sql_parsing/test_split_statements.py b/metadata-ingestion/tests/unit/sql_parsing/test_split_statements.py new file mode 100644 index 00000000000000..06e0e84ede5547 --- /dev/null +++ b/metadata-ingestion/tests/unit/sql_parsing/test_split_statements.py @@ -0,0 +1,51 @@ +from datahub.sql_parsing.split_statements import split_statements + + +def test_split_statements_complex() -> None: + test_sql = """ + CREATE TABLE Users (Id INT); + -- Comment here + INSERT INTO Users VALUES (1); + BEGIN + UPDATE Users SET Id = 2; + /* Multi-line + comment */ + DELETE FROM /* inline DELETE comment */ Users; + END + GO + SELECT * FROM Users + """ + + statements = [statement.strip() for statement in split_statements(test_sql)] + assert statements == [ + "CREATE TABLE Users (Id INT)", + "-- Comment here", + "INSERT INTO Users VALUES (1)", + "BEGIN", + "UPDATE Users SET Id = 2", + "/* Multi-line\n comment */", + "DELETE FROM /* inline DELETE comment */ Users", + "END", + "GO", + "SELECT * FROM Users", + ] + + +def test_split_statements_cte() -> None: + # SQL example from https://stackoverflow.com/a/11562724 + test_sql = """\ +WITH T AS +( SELECT InvoiceNumber, + DocTotal, + SUM(Sale + VAT) OVER(PARTITION BY InvoiceNumber) AS NewDocTotal + FROM PEDI_InvoiceDetail +) +-- comment +/* multi-line +comment */ +UPDATE T +SET DocTotal = NewDocTotal""" + statements = [statement.strip() for statement in split_statements(test_sql)] + assert statements == [ + test_sql, + ] From 970453e5841cb693f5bc2349b38b7d183424d27e Mon Sep 17 00:00:00 2001 From: Tim <50115603+bossenti@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:34:39 +0100 Subject: [PATCH 052/174] fix(ui): use correct docs link for csv enricher (#11917) --- datahub-web-react/src/app/ingest/source/builder/sources.json | 2 +- datahub-web-react/src/app/ingest/source/conf/csv/csv.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index c20869a1c849c2..5e81a7855e9c4c 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -284,7 +284,7 @@ "name": "csv-enricher", "displayName": "CSV", "description": "Import metadata from a formatted CSV.", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher", "recipe": "source: \n type: csv-enricher \n config: \n # URL of your csv file to ingest \n filename: \n array_delimiter: '|' \n delimiter: ',' \n write_semantics: PATCH" }, { diff --git a/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts b/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts index e1dc22c086fb43..e4cdee717923c2 100644 --- a/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts +++ b/datahub-web-react/src/app/ingest/source/conf/csv/csv.ts @@ -15,7 +15,7 @@ const csvConfig: SourceConfig = { type: 'csv-enricher', placeholderRecipe, displayName: 'CSV', - docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/csv', + docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher', logoUrl: csvLogo, }; From e7e208fdfe046e2fb10d4e0c11a930efca2b1330 Mon Sep 17 00:00:00 2001 From: kevinkarchacryl Date: Fri, 22 Nov 2024 13:55:09 -0500 Subject: [PATCH 053/174] fix(UI): incorrect month showing in MAU (#11918) --- .../analytics/resolver/GetChartsResolver.java | 18 +++++++++++++-- .../datahub/graphql/utils/DateUtilTest.java | 22 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java index 0fe6e5de0cac68..197ac87c1e22d8 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetChartsResolver.java @@ -84,8 +84,21 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart( final DateTime end, final String title, final DateInterval interval) { - final DateRange dateRange = - new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis())); + + final DateRange dateRange; + + // adjust month to show 1st of month rather than last day of previous month + if (interval == DateInterval.MONTH) { + dateRange = + new DateRange( + String.valueOf(beginning.plusDays(1).getMillis()), // Shift start by 1 day + String.valueOf(end.plusDays(1).getMillis()) // Shift end by 1 day + ); + } else { + // week display starting Sundays + dateRange = + new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis())); + } final List timeSeriesLines = _analyticsService.getTimeseriesChart( @@ -96,6 +109,7 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart( ImmutableMap.of(), Collections.emptyMap(), Optional.of("browserId")); + return TimeSeriesChart.builder() .setTitle(title) .setDateRange(dateRange) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java index 6ecbc8d015b29a..4383df9d46a4bc 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/utils/DateUtilTest.java @@ -47,4 +47,26 @@ public void testStartOfNextWeek() { Mockito.when(dateUtil.getNow()).thenReturn(setTimeParts(8, false)); assertEqualStartOfNextWeek(dateUtil, 9); } + + // validates logic to display correct dates in MAU chart + @Test + public void testDateAdjustmentsForMonth() { + DateUtil dateUtil = Mockito.spy(DateUtil.class); + + Mockito.when(dateUtil.getNow()).thenReturn(new DateTime(2024, 11, 15, 0, 0, 0)); + + // start date should be next month minus a day + // but we want to display Dec 1 instead of Nov 30, so add a day and verify it's Dec + DateTime startOfNextMonthMinus12 = dateUtil.getStartOfNextMonth().minusMonths(12); + DateTime adjustedStart = startOfNextMonthMinus12.minusMillis(1).plusDays(1); + assertEquals(12, adjustedStart.getMonthOfYear()); // Verify it is December + assertEquals(2023, adjustedStart.getYear()); // Verify it is 2023 + + // verify that the end date displays correctly + // the chart will display Oct 1 as the last month because we don't show current month + DateTime startOfThisMonth = dateUtil.getStartOfThisMonth(); + DateTime adjustedEnd = startOfThisMonth.minusMillis(1).plusDays(1); + assertEquals(11, adjustedEnd.getMonthOfYear()); // Verify it is November + assertEquals(2024, adjustedEnd.getYear()); // Verify it is 2024 + } } From 28cc8caf65498c37bc19693a0d8cd872426ca18b Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 22 Nov 2024 16:31:13 -0600 Subject: [PATCH 054/174] fix(batch-patch): fix patches in batches (#11928) --- .../metadata/aspect/batch/AspectsBatch.java | 39 ++- .../entity/ebean/batch/AspectsBatchImpl.java | 11 +- .../ebean/batch/AspectsBatchImplTest.java | 17 +- .../metadata/entity/EntityServiceImpl.java | 239 ++++++++++-------- .../entity/EbeanEntityServiceTest.java | 234 +++++++++++++++++ .../metadata/entity/EntityServiceTest.java | 14 +- .../metadata/entity/EntityService.java | 2 + 7 files changed, 425 insertions(+), 131 deletions(-) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java index dc7934ad5cc193..30f5dce379a077 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java @@ -9,6 +9,7 @@ import com.linkedin.util.Pair; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -49,7 +50,8 @@ default List getMCPItems() { * various hooks */ Pair>, List> toUpsertBatchItems( - Map> latestAspects); + Map> latestAspects, + Map> nextVersions); /** * Apply read mutations to batch @@ -227,4 +229,39 @@ static String toAbbreviatedString(Collection items, int max + StringUtils.abbreviate(itemsAbbreviated.toString(), maxWidth) + '}'; } + + /** + * Increment aspect within a batch, tracking both the next aspect version and the most recent + * + * @param changeMCP changeMCP to be incremented + * @param latestAspects lastest aspects within the batch + * @param nextVersions next version for the aspects in the batch + * @return the incremented changeMCP + */ + static ChangeMCP incrementBatchVersion( + ChangeMCP changeMCP, + Map> latestAspects, + Map> nextVersions) { + long nextVersion = + nextVersions + .getOrDefault(changeMCP.getUrn().toString(), Collections.emptyMap()) + .getOrDefault(changeMCP.getAspectName(), 0L); + + changeMCP.setPreviousSystemAspect( + latestAspects + .getOrDefault(changeMCP.getUrn().toString(), Collections.emptyMap()) + .getOrDefault(changeMCP.getAspectName(), null)); + + changeMCP.setNextAspectVersion(nextVersion); + + // support inner-batch upserts + latestAspects + .computeIfAbsent(changeMCP.getUrn().toString(), key -> new HashMap<>()) + .put(changeMCP.getAspectName(), changeMCP.getSystemAspect(nextVersion)); + nextVersions + .computeIfAbsent(changeMCP.getUrn().toString(), key -> new HashMap<>()) + .put(changeMCP.getAspectName(), nextVersion + 1); + + return changeMCP; + } } diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java index 7f56abe64f9a77..c0d65640df2378 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java @@ -47,7 +47,8 @@ public class AspectsBatchImpl implements AspectsBatch { */ @Override public Pair>, List> toUpsertBatchItems( - final Map> latestAspects) { + Map> latestAspects, + Map> nextVersions) { // Process proposals to change items Stream mutatedProposalsStream = @@ -56,6 +57,7 @@ public Pair>, List> toUpsertBatchItems( .filter(item -> item instanceof ProposedItem) .map(item -> (MCPItem) item) .collect(Collectors.toList())); + // Regular change items Stream changeMCPStream = items.stream().filter(item -> !(item instanceof ProposedItem)); @@ -83,10 +85,8 @@ public Pair>, List> toUpsertBatchItems( currentValue, retrieverContext.getAspectRetriever()); } - // Populate old aspect for write hooks - upsertItem.setPreviousSystemAspect(latest); - - return upsertItem; + return AspectsBatch.incrementBatchVersion( + upsertItem, latestAspects, nextVersions); }) .collect(Collectors.toCollection(LinkedList::new)); @@ -96,6 +96,7 @@ public Pair>, List> toUpsertBatchItems( LinkedList newItems = applyMCPSideEffects(upsertBatchItems).collect(Collectors.toCollection(LinkedList::new)); upsertBatchItems.addAll(newItems); + Map> newUrnAspectNames = getNewUrnAspectsMap(getUrnAspectsMap(), upsertBatchItems); diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java index 31dd868b4cb4a3..96f535f2295aa4 100644 --- a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java +++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java @@ -41,6 +41,7 @@ import io.datahubproject.metadata.context.RetrieverContext; import java.nio.charset.StandardCharsets; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -120,7 +121,7 @@ public void toUpsertBatchItemsChangeItemTest() { AspectsBatchImpl.builder().items(testItems).retrieverContext(retrieverContext).build(); assertEquals( - testBatch.toUpsertBatchItems(Map.of()), + testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()), Pair.of(Map.of(), testItems), "Expected noop, pass through with no additional MCPs or changes"); } @@ -176,7 +177,7 @@ public void toUpsertBatchItemsPatchItemTest() { AspectsBatchImpl.builder().items(testItems).retrieverContext(retrieverContext).build(); assertEquals( - testBatch.toUpsertBatchItems(Map.of()), + testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()), Pair.of( Map.of(), List.of( @@ -195,7 +196,7 @@ public void toUpsertBatchItemsPatchItemTest() { .recordTemplate( new StructuredProperties() .setProperties(new StructuredPropertyValueAssignmentArray())) - .systemMetadata(testItems.get(0).getSystemMetadata()) + .systemMetadata(testItems.get(0).getSystemMetadata().setVersion("1")) .build(mockAspectRetriever), ChangeItemImpl.builder() .urn( @@ -212,7 +213,7 @@ public void toUpsertBatchItemsPatchItemTest() { .recordTemplate( new StructuredProperties() .setProperties(new StructuredPropertyValueAssignmentArray())) - .systemMetadata(testItems.get(1).getSystemMetadata()) + .systemMetadata(testItems.get(1).getSystemMetadata().setVersion("1")) .build(mockAspectRetriever))), "Expected patch items converted to upsert change items"); } @@ -264,7 +265,7 @@ public void toUpsertBatchItemsProposedItemTest() { AspectsBatchImpl.builder().items(testItems).retrieverContext(retrieverContext).build(); assertEquals( - testBatch.toUpsertBatchItems(Map.of()), + testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()), Pair.of( Map.of(), List.of( @@ -280,7 +281,7 @@ public void toUpsertBatchItemsProposedItemTest() { .getEntitySpec(DATASET_ENTITY_NAME) .getAspectSpec(STATUS_ASPECT_NAME)) .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .systemMetadata(testItems.get(0).getSystemMetadata()) + .systemMetadata(testItems.get(0).getSystemMetadata().setVersion("1")) .recordTemplate(new Status().setRemoved(false)) .build(mockAspectRetriever), ChangeItemImpl.builder() @@ -295,7 +296,7 @@ public void toUpsertBatchItemsProposedItemTest() { .getEntitySpec(DATASET_ENTITY_NAME) .getAspectSpec(STATUS_ASPECT_NAME)) .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .systemMetadata(testItems.get(1).getSystemMetadata()) + .systemMetadata(testItems.get(1).getSystemMetadata().setVersion("1")) .recordTemplate(new Status().setRemoved(false)) .build(mockAspectRetriever))), "Mutation to status aspect"); @@ -328,7 +329,7 @@ public void singleInvalidDoesntBreakBatch() { .build(); assertEquals( - testBatch.toUpsertBatchItems(Map.of()).getSecond().size(), + testBatch.toUpsertBatchItems(new HashMap<>(), new HashMap<>()).getSecond().size(), 1, "Expected 1 valid mcp to be passed through."); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 9337ea3c2b6f77..a0a55cf505cf35 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -868,71 +868,64 @@ private List ingestAspectsToLocalDB( // Read before write is unfortunate, however batch it final Map> urnAspects = batchWithDefaults.getUrnAspectsMap(); // read #1 - final Map> latestAspects = + Map> databaseAspects = + aspectDao.getLatestAspects(urnAspects, true); + + final Map> batchAspects = EntityUtils.toSystemAspects( - opContext.getRetrieverContext().get(), - aspectDao.getLatestAspects(urnAspects, true)); + opContext.getRetrieverContext().get(), databaseAspects); + // read #2 (potentially) final Map> nextVersions = - EntityUtils.calculateNextVersions( - txContext, aspectDao, latestAspects, urnAspects); + EntityUtils.calculateNextVersions(txContext, aspectDao, batchAspects, urnAspects); // 1. Convert patches to full upserts // 2. Run any entity/aspect level hooks Pair>, List> updatedItems = - batchWithDefaults.toUpsertBatchItems(latestAspects); + batchWithDefaults.toUpsertBatchItems(batchAspects, nextVersions); // Fetch additional information if needed - final Map> updatedLatestAspects; - final Map> updatedNextVersions; + final List changeMCPs; + if (!updatedItems.getFirst().isEmpty()) { + // These items are new items from side effects + Map> sideEffects = updatedItems.getFirst(); + + final Map> updatedLatestAspects; + final Map> updatedNextVersions; + Map> newLatestAspects = EntityUtils.toSystemAspects( opContext.getRetrieverContext().get(), aspectDao.getLatestAspects(updatedItems.getFirst(), true)); // merge - updatedLatestAspects = AspectsBatch.merge(latestAspects, newLatestAspects); + updatedLatestAspects = AspectsBatch.merge(batchAspects, newLatestAspects); Map> newNextVersions = EntityUtils.calculateNextVersions( txContext, aspectDao, updatedLatestAspects, updatedItems.getFirst()); // merge updatedNextVersions = AspectsBatch.merge(nextVersions, newNextVersions); + + changeMCPs = + updatedItems.getSecond().stream() + .peek( + changeMCP -> { + // Add previous version to each side-effect + if (sideEffects + .getOrDefault( + changeMCP.getUrn().toString(), Collections.emptySet()) + .contains(changeMCP.getAspectName())) { + + AspectsBatch.incrementBatchVersion( + changeMCP, updatedLatestAspects, updatedNextVersions); + } + }) + .collect(Collectors.toList()); } else { - updatedLatestAspects = latestAspects; - updatedNextVersions = nextVersions; + changeMCPs = updatedItems.getSecond(); } - // Add previous version to each upsert - List changeMCPs = - updatedItems.getSecond().stream() - .peek( - changeMCP -> { - String urnStr = changeMCP.getUrn().toString(); - long nextVersion = - updatedNextVersions - .getOrDefault(urnStr, Map.of()) - .getOrDefault(changeMCP.getAspectName(), 0L); - - changeMCP.setPreviousSystemAspect( - updatedLatestAspects - .getOrDefault(urnStr, Map.of()) - .getOrDefault(changeMCP.getAspectName(), null)); - - changeMCP.setNextAspectVersion(nextVersion); - - // support inner-batch upserts - updatedLatestAspects - .computeIfAbsent(urnStr, key -> new HashMap<>()) - .put( - changeMCP.getAspectName(), - changeMCP.getSystemAspect(nextVersion)); - updatedNextVersions - .computeIfAbsent(urnStr, key -> new HashMap<>()) - .put(changeMCP.getAspectName(), nextVersion + 1); - }) - .collect(Collectors.toList()); - // No changes, return if (changeMCPs.isEmpty()) { return Collections.emptyList(); @@ -954,40 +947,50 @@ private List ingestAspectsToLocalDB( List upsertResults = changeMCPs.stream() .map( - item -> { - final EntityAspect.EntitySystemAspect latest = - (EntityAspect.EntitySystemAspect) item.getPreviousSystemAspect(); + writeItem -> { + + /* + database*Aspect - should be used for comparisons of before batch operation information + */ + final EntityAspect databaseAspect = + databaseAspects + .getOrDefault(writeItem.getUrn().toString(), Map.of()) + .get(writeItem.getAspectName()); + final EntityAspect.EntitySystemAspect databaseSystemAspect = + databaseAspect == null + ? null + : EntityAspect.EntitySystemAspect.builder() + .build( + writeItem.getEntitySpec(), + writeItem.getAspectSpec(), + databaseAspect); final UpdateAspectResult result; - if (overwrite || latest == null) { + /* + This condition is specifically for an older conditional write ingestAspectIfNotPresent() + overwrite is always true otherwise + */ + if (overwrite || databaseAspect == null) { result = - ingestAspectToLocalDB( - txContext, - item.getUrn(), - item.getAspectName(), - item.getRecordTemplate(), - item.getAuditStamp(), - item.getSystemMetadata(), - latest == null ? null : latest, - item.getNextAspectVersion()) + ingestAspectToLocalDB(txContext, writeItem, databaseSystemAspect) .toBuilder() - .request(item) + .request(writeItem) .build(); } else { - RecordTemplate oldValue = latest.getRecordTemplate(); - SystemMetadata oldMetadata = latest.getSystemMetadata(); + RecordTemplate oldValue = databaseSystemAspect.getRecordTemplate(); + SystemMetadata oldMetadata = databaseSystemAspect.getSystemMetadata(); result = UpdateAspectResult.builder() - .urn(item.getUrn()) - .request(item) + .urn(writeItem.getUrn()) + .request(writeItem) .oldValue(oldValue) .newValue(oldValue) .oldSystemMetadata(oldMetadata) .newSystemMetadata(oldMetadata) .operation(MetadataAuditOperation.UPDATE) - .auditStamp(item.getAuditStamp()) - .maxVersion(latest.getVersion()) + .auditStamp(writeItem.getAuditStamp()) + .maxVersion(databaseAspect.getVersion()) .build(); } @@ -1011,8 +1014,8 @@ private List ingestAspectsToLocalDB( // Only consider retention when there was a previous version .filter( result -> - latestAspects.containsKey(result.getUrn().toString()) - && latestAspects + batchAspects.containsKey(result.getUrn().toString()) + && batchAspects .get(result.getUrn().toString()) .containsKey(result.getRequest().getAspectName())) .filter( @@ -1102,9 +1105,11 @@ private List emitMCL( * @param auditStamp an {@link AuditStamp} containing metadata about the writer & current time * @param systemMetadata * @return the {@link RecordTemplate} representation of the written aspect object + * @deprecated See Conditional Write ChangeType CREATE */ @Nullable @Override + @Deprecated public RecordTemplate ingestAspectIfNotPresent( @Nonnull OperationContext opContext, @Nonnull Urn urn, @@ -2495,87 +2500,107 @@ private Map getEnvelopedAspects( ((EntityAspect.EntitySystemAspect) systemAspect).toEnvelopedAspects())); } + /** + * @param txContext Transaction context, keeps track of retries, exceptions etc. + * @param writeItem The aspect being written + * @param databaseAspect The aspect as it exists in the database. + * @return result object + */ @Nonnull private UpdateAspectResult ingestAspectToLocalDB( @Nullable TransactionContext txContext, - @Nonnull final Urn urn, - @Nonnull final String aspectName, - @Nonnull final RecordTemplate newValue, - @Nonnull final AuditStamp auditStamp, - @Nonnull final SystemMetadata providedSystemMetadata, - @Nullable final EntityAspect.EntitySystemAspect latest, - @Nonnull final Long nextVersion) { + @Nonnull final ChangeMCP writeItem, + @Nullable final EntityAspect.EntitySystemAspect databaseAspect) { // Set the "last run id" to be the run id provided with the new system metadata. This will be // stored in index // for all aspects that have a run id, regardless of whether they change. - providedSystemMetadata.setLastRunId( - providedSystemMetadata.getRunId(GetMode.NULL), SetMode.IGNORE_NULL); + writeItem + .getSystemMetadata() + .setLastRunId(writeItem.getSystemMetadata().getRunId(GetMode.NULL), SetMode.IGNORE_NULL); // 2. Compare the latest existing and new. - final RecordTemplate oldValue = latest == null ? null : latest.getRecordTemplate(); + final EntityAspect.EntitySystemAspect previousBatchAspect = + (EntityAspect.EntitySystemAspect) writeItem.getPreviousSystemAspect(); + final RecordTemplate previousValue = + previousBatchAspect == null ? null : previousBatchAspect.getRecordTemplate(); // 3. If there is no difference between existing and new, we just update // the lastObserved in system metadata. RunId should stay as the original runId - if (oldValue != null && DataTemplateUtil.areEqual(oldValue, newValue)) { - SystemMetadata latestSystemMetadata = latest.getSystemMetadata(); - latestSystemMetadata.setLastObserved(providedSystemMetadata.getLastObserved()); + if (previousValue != null + && DataTemplateUtil.areEqual(previousValue, writeItem.getRecordTemplate())) { + + SystemMetadata latestSystemMetadata = previousBatchAspect.getSystemMetadata(); + latestSystemMetadata.setLastObserved(writeItem.getSystemMetadata().getLastObserved()); latestSystemMetadata.setLastRunId( - providedSystemMetadata.getLastRunId(GetMode.NULL), SetMode.IGNORE_NULL); + writeItem.getSystemMetadata().getLastRunId(GetMode.NULL), SetMode.IGNORE_NULL); - latest.getEntityAspect().setSystemMetadata(RecordUtils.toJsonString(latestSystemMetadata)); + previousBatchAspect + .getEntityAspect() + .setSystemMetadata(RecordUtils.toJsonString(latestSystemMetadata)); - log.info("Ingesting aspect with name {}, urn {}", aspectName, urn); - aspectDao.saveAspect(txContext, latest.getEntityAspect(), false); + log.info( + "Ingesting aspect with name {}, urn {}", + previousBatchAspect.getAspectName(), + previousBatchAspect.getUrn()); + aspectDao.saveAspect(txContext, previousBatchAspect.getEntityAspect(), false); // metrics aspectDao.incrementWriteMetrics( - aspectName, 1, latest.getMetadataRaw().getBytes(StandardCharsets.UTF_8).length); + previousBatchAspect.getAspectName(), + 1, + previousBatchAspect.getMetadataRaw().getBytes(StandardCharsets.UTF_8).length); return UpdateAspectResult.builder() - .urn(urn) - .oldValue(oldValue) - .newValue(oldValue) - .oldSystemMetadata(latest.getSystemMetadata()) + .urn(writeItem.getUrn()) + .oldValue(previousValue) + .newValue(previousValue) + .oldSystemMetadata(previousBatchAspect.getSystemMetadata()) .newSystemMetadata(latestSystemMetadata) .operation(MetadataAuditOperation.UPDATE) - .auditStamp(auditStamp) + .auditStamp(writeItem.getAuditStamp()) .maxVersion(0) .build(); } // 4. Save the newValue as the latest version - log.debug("Ingesting aspect with name {}, urn {}", aspectName, urn); - String newValueStr = EntityApiUtils.toJsonAspect(newValue); + log.debug( + "Ingesting aspect with name {}, urn {}", writeItem.getAspectName(), writeItem.getUrn()); + String newValueStr = EntityApiUtils.toJsonAspect(writeItem.getRecordTemplate()); long versionOfOld = aspectDao.saveLatestAspect( txContext, - urn.toString(), - aspectName, - latest == null ? null : EntityApiUtils.toJsonAspect(oldValue), - latest == null ? null : latest.getCreatedBy(), - latest == null ? null : latest.getEntityAspect().getCreatedFor(), - latest == null ? null : latest.getCreatedOn(), - latest == null ? null : latest.getSystemMetadataRaw(), + writeItem.getUrn().toString(), + writeItem.getAspectName(), + previousBatchAspect == null ? null : EntityApiUtils.toJsonAspect(previousValue), + previousBatchAspect == null ? null : previousBatchAspect.getCreatedBy(), + previousBatchAspect == null + ? null + : previousBatchAspect.getEntityAspect().getCreatedFor(), + previousBatchAspect == null ? null : previousBatchAspect.getCreatedOn(), + previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadataRaw(), newValueStr, - auditStamp.getActor().toString(), - auditStamp.hasImpersonator() ? auditStamp.getImpersonator().toString() : null, - new Timestamp(auditStamp.getTime()), - EntityApiUtils.toJsonAspect(providedSystemMetadata), - nextVersion); + writeItem.getAuditStamp().getActor().toString(), + writeItem.getAuditStamp().hasImpersonator() + ? writeItem.getAuditStamp().getImpersonator().toString() + : null, + new Timestamp(writeItem.getAuditStamp().getTime()), + EntityApiUtils.toJsonAspect(writeItem.getSystemMetadata()), + writeItem.getNextAspectVersion()); // metrics aspectDao.incrementWriteMetrics( - aspectName, 1, newValueStr.getBytes(StandardCharsets.UTF_8).length); + writeItem.getAspectName(), 1, newValueStr.getBytes(StandardCharsets.UTF_8).length); return UpdateAspectResult.builder() - .urn(urn) - .oldValue(oldValue) - .newValue(newValue) - .oldSystemMetadata(latest == null ? null : latest.getSystemMetadata()) - .newSystemMetadata(providedSystemMetadata) + .urn(writeItem.getUrn()) + .oldValue(previousValue) + .newValue(writeItem.getRecordTemplate()) + .oldSystemMetadata( + previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadata()) + .newSystemMetadata(writeItem.getSystemMetadata()) .operation(MetadataAuditOperation.UPDATE) - .auditStamp(auditStamp) + .auditStamp(writeItem.getAuditStamp()) .maxVersion(versionOfOld) .build(); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index 04c9297b1ed7aa..f2ed2fddba7654 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -1,6 +1,8 @@ package com.linkedin.metadata.entity; import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static com.linkedin.metadata.Constants.GLOBAL_TAGS_ASPECT_NAME; import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; @@ -8,7 +10,11 @@ import static org.testng.Assert.assertTrue; import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; import com.linkedin.common.Status; +import com.linkedin.common.TagAssociation; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.DataTemplateUtil; @@ -18,17 +24,21 @@ import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.Constants; import com.linkedin.metadata.EbeanTestUtils; +import com.linkedin.metadata.aspect.patch.GenericJsonPatch; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import com.linkedin.metadata.config.EbeanConfiguration; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; +import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.service.UpdateIndicesService; +import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; @@ -433,6 +443,220 @@ public void testBatchDuplicate() throws Exception { "Expected 2nd item to be the latest"); } + @Test + public void testBatchPatchWithTrailingNoOp() throws Exception { + Urn entityUrn = + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchWithTrailingNoOp,PROD)"); + TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); + Urn tag2 = UrnUtils.getUrn("urn:li:tag:tag2"); + Urn tagOther = UrnUtils.getUrn("urn:li:tag:other"); + + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + + ChangeItemImpl initialAspectTag1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .recordTemplate( + new GlobalTags() + .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1)))) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + + PatchItemImpl patchAdd2 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + PatchItemImpl patchRemoveNonExistent = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.REMOVE, tagOther))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + // establish base entity + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(initialAspectTag1)) + .build(), + false, + true); + + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchAdd2, patchRemoveNonExistent)) + .build(), + false, + true); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals( + envelopedAspect.getSystemMetadata().getVersion(), + "2", + "Expected version 2. 1 - Initial, + 1 batch operation (1 add, 1 remove)"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Set.of(tag1, tag2), + "Expected both tags"); + } + + @Test + public void testBatchPatchAdd() throws Exception { + Urn entityUrn = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)"); + TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); + TagUrn tag2 = TagUrn.createFromString("urn:li:tag:tag2"); + TagUrn tag3 = TagUrn.createFromString("urn:li:tag:tag3"); + + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + + ChangeItemImpl initialAspectTag1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .recordTemplate( + new GlobalTags() + .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1)))) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + + PatchItemImpl patchAdd3 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag3))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + PatchItemImpl patchAdd2 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + PatchItemImpl patchAdd1 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag1))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + // establish base entity + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(initialAspectTag1)) + .build(), + false, + true); + + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchAdd3, patchAdd2, patchAdd1)) + .build(), + false, + true); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "3", "Expected version 3"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Set.of(tag1, tag2, tag3), + "Expected all tags"); + } + @Test public void dataGeneratorThreadingTest() { DataGenerator dataGenerator = new DataGenerator(opContext, _entityServiceImpl); @@ -659,4 +883,14 @@ public void run() { } } } + + private static GenericJsonPatch.PatchOp tagPatchOp(PatchOperationType op, Urn tagUrn) { + GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp(); + patchOp.setOp(op.getValue()); + patchOp.setPath(String.format("/tags/%s", tagUrn)); + if (PatchOperationType.ADD.equals(op)) { + patchOp.setValue(Map.of("tag", tagUrn.toString())); + } + return patchOp; + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index 53f5ebfe59728e..d704037b39911e 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -50,10 +50,8 @@ import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.models.AspectSpec; -import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistryException; -import com.linkedin.metadata.models.registry.MergedEntityRegistry; import com.linkedin.metadata.run.AspectRowSummary; import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.snapshot.CorpUserSnapshot; @@ -75,6 +73,7 @@ import com.linkedin.structured.StructuredPropertyValueAssignmentArray; import com.linkedin.util.Pair; import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; import jakarta.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; @@ -113,18 +112,13 @@ public abstract class EntityServiceTest ingestAspects( * @param auditStamp an {@link AuditStamp} containing metadata about the writer & current time * @param systemMetadata * @return the {@link RecordTemplate} representation of the written aspect object + * @deprecated See Conditional Write ChangeType CREATE */ + @Deprecated RecordTemplate ingestAspectIfNotPresent( @Nonnull OperationContext opContext, @Nonnull Urn urn, From 2878c65ceb6e7df6ffb7b70b0869a234d9d0280b Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 22 Nov 2024 16:32:33 -0600 Subject: [PATCH 055/174] =?UTF-8?q?fix(structuredProps)=20Add=20validation?= =?UTF-8?q?=20that=20ID=20and=20qualifiedName=20have=20no=E2=80=A6=20(#119?= =?UTF-8?q?30)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../PropertyDefinitionValidator.java | 19 ++++++++++ .../PropertyDefinitionValidatorTest.java | 36 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java index ae5472af622ad5..6e047c12da9a9f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/structuredproperties/validation/PropertyDefinitionValidator.java @@ -89,6 +89,9 @@ public static Stream validateDefinitionUpserts( item.getAspect(StructuredPropertyDefinition.class); versionFormatCheck(item, newDefinition.getVersion()).ifPresent(exceptions::addException); + urnIdCheck(item).ifPresent(exceptions::addException); + qualifiedNameCheck(item, newDefinition.getQualifiedName()) + .ifPresent(exceptions::addException); if (item.getPreviousSystemAspect() != null) { @@ -192,4 +195,20 @@ private static Optional versionFormatCheck( } return Optional.empty(); } + + private static Optional urnIdCheck(MCPItem item) { + if (item.getUrn().getId().contains(" ")) { + return Optional.of(AspectValidationException.forItem(item, "Urn ID cannot have spaces")); + } + return Optional.empty(); + } + + private static Optional qualifiedNameCheck( + MCPItem item, @Nonnull String qualifiedName) { + if (qualifiedName.contains(" ")) { + return Optional.of( + AspectValidationException.forItem(item, "Qualified names cannot have spaces")); + } + return Optional.empty(); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java index 2af731a51145e3..18949f0566dd19 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/structuredproperties/validators/PropertyDefinitionValidatorTest.java @@ -397,4 +397,40 @@ public void testCanChangeAllowedValueDescriptions() .count(), 0); } + + @Test + public void testUrnIdWithSpace() + throws URISyntaxException, CloneNotSupportedException, AspectValidationException { + Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:test me out.foo.bar"); + StructuredPropertyDefinition newProperty = new StructuredPropertyDefinition(); + newProperty.setEntityTypes(new UrnArray(Urn.createFromString("urn:li:logicalEntity:dataset"))); + newProperty.setDisplayName("oldProp"); + newProperty.setQualifiedName("foo.bar"); + newProperty.setCardinality(PropertyCardinality.MULTIPLE); + newProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + assertEquals( + PropertyDefinitionValidator.validateDefinitionUpserts( + TestMCP.ofOneMCP(propertyUrn, null, newProperty, entityRegistry), + mockRetrieverContext) + .count(), + 1); + } + + @Test + public void testQualifiedNameWithSpace() + throws URISyntaxException, CloneNotSupportedException, AspectValidationException { + Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:foo.bar"); + StructuredPropertyDefinition newProperty = new StructuredPropertyDefinition(); + newProperty.setEntityTypes(new UrnArray(Urn.createFromString("urn:li:logicalEntity:dataset"))); + newProperty.setDisplayName("oldProp"); + newProperty.setQualifiedName("foo.bar with spaces"); + newProperty.setCardinality(PropertyCardinality.MULTIPLE); + newProperty.setValueType(Urn.createFromString("urn:li:logicalType:STRING")); + assertEquals( + PropertyDefinitionValidator.validateDefinitionUpserts( + TestMCP.ofOneMCP(propertyUrn, null, newProperty, entityRegistry), + mockRetrieverContext) + .count(), + 1); + } } From 794ac2b4e827a7bd7713d0d6d9edf5001f8d848d Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 22 Nov 2024 23:47:56 -0600 Subject: [PATCH 056/174] fix(rest.li): fix use of Criterion in rest.li filters (#11932) --- docs/how/updating-datahub.md | 2 +- docs/managed-datahub/release-notes/v_0_3_7.md | 2 +- .../java/com/linkedin/metadata/Constants.java | 1 + .../resources/analytics/Analytics.java | 12 +- .../resources/entity/AspectResource.java | 8 +- .../resources/entity/EntityResource.java | 16 +- .../metadata/utils/CriterionUtils.java | 64 ++++ .../metadata/utils/CriterionUtilsTest.java | 274 ++++++++++++++++++ 8 files changed, 357 insertions(+), 22 deletions(-) create mode 100644 metadata-utils/src/test/java/com/linkedin/metadata/utils/CriterionUtilsTest.java diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index c21d197de29f08..10173ccd4a1de4 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -38,7 +38,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes -- #11486 - Deprecated Criterion filters using `value`. Use `values` instead. This also deprecates the ability to use comma delimited string to represent multiple values using `value`. +- #11486 - Criterion's `value` parameter has been previously deprecated. Use of `value` instead of `values` is no longer supported and will be completely removed on the next major version. - #11484 - Metadata service authentication enabled by default - #11484 - Rest API authorization enabled by default - #10472 - `SANDBOX` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases. diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index 19cb04e9f56039..a1514629768852 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -32,7 +32,7 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies datahub: timezone: 'America/Los_Angeles' ``` - - #11486 - Deprecated Criterion filters using `value`. Use `values` instead. This also deprecates the ability to use comma delimited string to represent multiple values using `value`. + - #11486 - Criterion's `value` parameter has been previously deprecated. Use of `value` instead of `values` is no longer supported and will be completely removed on the next major version. - #10472 - `SANDBOX` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases. - #11619 - schema field/column paths can no longer be empty strings - #11619 - schema field/column paths can no longer be duplicated within the schema diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index f1f096640bc216..077e0e2b666be1 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -10,6 +10,7 @@ public class Constants { public static final String INTERNAL_DELEGATED_FOR_ACTOR_HEADER_NAME = "X-DataHub-Delegated-For"; public static final String INTERNAL_DELEGATED_FOR_ACTOR_TYPE = "X-DataHub-Delegated-For-"; + public static final String URN_LI_PREFIX = "urn:li:"; public static final String DATAHUB_ACTOR = "urn:li:corpuser:datahub"; // Super user. public static final String SYSTEM_ACTOR = "urn:li:corpuser:__datahub_system"; // DataHub internal service principal. diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java index 9bbe1bb35fc654..94da6308eda1f2 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/analytics/Analytics.java @@ -5,7 +5,6 @@ import com.datahub.authorization.AuthUtil; import com.datahub.plugins.auth.authorization.Authorizer; import com.linkedin.analytics.GetTimeseriesAggregatedStatsResponse; -import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.resources.restli.RestliUtils; import com.linkedin.metadata.timeseries.TimeseriesAspectService; @@ -14,7 +13,6 @@ import com.linkedin.restli.server.RestLiServiceException; import com.linkedin.restli.server.annotations.Action; import com.linkedin.restli.server.annotations.ActionParam; -import com.linkedin.restli.server.annotations.Context; import com.linkedin.restli.server.annotations.Optional; import com.linkedin.restli.server.annotations.RestLiSimpleResource; import com.linkedin.restli.server.resources.SimpleResourceTemplate; @@ -24,12 +22,10 @@ import com.linkedin.timeseries.GroupingBucket; import com.linkedin.timeseries.GroupingBucketArray; import java.util.Arrays; -import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.inject.Inject; import javax.inject.Named; -import javax.servlet.http.HttpServletRequest; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.RequestContext; @@ -38,6 +34,7 @@ import static com.datahub.authorization.AuthUtil.isAPIAuthorized; import static com.linkedin.metadata.authorization.ApiGroup.TIMESERIES; import static com.linkedin.metadata.authorization.ApiOperation.READ; +import static com.linkedin.metadata.utils.CriterionUtils.validateAndConvert; /** Rest.li entry point: /analytics */ @Slf4j @@ -90,8 +87,9 @@ public Task getTimeseriesStats( resp.setEntityName(entityName); resp.setAspectName(aspectName); resp.setAggregationSpecs(new AggregationSpecArray(Arrays.asList(aggregationSpecs))); - if (filter != null) { - resp.setFilter(filter); + final Filter finalFilter = validateAndConvert(filter); + if (finalFilter != null) { + resp.setFilter(finalFilter); } if (groupingBuckets != null) { resp.setGroupingBuckets(new GroupingBucketArray(Arrays.asList(groupingBuckets))); @@ -99,7 +97,7 @@ public Task getTimeseriesStats( GenericTable aggregatedStatsTable = timeseriesAspectService.getAggregatedStats(opContext, - entityName, aspectName, aggregationSpecs, filter, groupingBuckets); + entityName, aspectName, aggregationSpecs, finalFilter, groupingBuckets); resp.setTable(aggregatedStatsTable); return resp; }); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index 37dca1cecd817c..a8b9c34ab66ae6 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -11,6 +11,7 @@ import static com.linkedin.metadata.authorization.ApiOperation.READ; import static com.linkedin.metadata.resources.operations.OperationsResource.*; import static com.linkedin.metadata.resources.restli.RestliConstants.*; +import static com.linkedin.metadata.utils.CriterionUtils.validateAndConvert; import com.codahale.metrics.MetricRegistry; import com.datahub.authentication.Authentication; @@ -22,14 +23,12 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.aspect.EnvelopedAspectArray; import com.linkedin.metadata.aspect.VersionedAspect; -import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.authorization.PoliciesConfig; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.IngestResult; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.entity.validation.ValidationException; -import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; import com.linkedin.metadata.resources.operations.Utils; @@ -38,7 +37,6 @@ import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.MetadataChangeProposal; -import com.linkedin.mxe.SystemMetadata; import com.linkedin.parseq.Task; import com.linkedin.restli.common.HttpStatus; import com.linkedin.restli.internal.server.methods.AnyRecord; @@ -59,8 +57,6 @@ import java.time.Clock; import java.util.Arrays; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -239,7 +235,7 @@ public Task getTimeseriesAspectValues( startTimeMillis, endTimeMillis, limit, - filter, + validateAndConvert(filter), sort))); return response; }, diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 30aa3ffa578c17..6c5576f2e5d9f4 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -12,6 +12,7 @@ import static com.linkedin.metadata.entity.validation.ValidationUtils.*; import static com.linkedin.metadata.resources.restli.RestliConstants.*; import static com.linkedin.metadata.search.utils.SearchUtils.*; +import static com.linkedin.metadata.utils.CriterionUtils.validateAndConvert; import static com.linkedin.metadata.utils.PegasusUtils.*; import static com.linkedin.metadata.utils.SystemMetadataUtils.generateSystemMetadataIfEmpty; @@ -401,7 +402,7 @@ public Task search( // This API is not used by the frontend for search bars so we default to structured result = entitySearchService.search(opContext, - List.of(entityName), input, filter, sortCriterionList, start, count); + List.of(entityName), input, validateAndConvert(filter), sortCriterionList, start, count); if (!isAPIAuthorizedResult( opContext, @@ -448,7 +449,7 @@ public Task searchAcrossEntities( log.info("GET SEARCH RESULTS ACROSS ENTITIES for {} with query {}", entityList, input); return RestliUtils.toTask( () -> { - SearchResult result = searchService.searchAcrossEntities(opContext, entityList, input, filter, sortCriterionList, start, count); + SearchResult result = searchService.searchAcrossEntities(opContext, entityList, input, validateAndConvert(filter), sortCriterionList, start, count); if (!isAPIAuthorizedResult( opContext, result)) { @@ -514,7 +515,7 @@ public Task scrollAcrossEntities( opContext, entityList, input, - filter, + validateAndConvert(filter), sortCriterionList, scrollId, keepAlive, @@ -583,7 +584,7 @@ public Task searchAcrossLineage( entityList, input, maxHops, - filter, + validateAndConvert(filter), sortCriterionList, start, count), @@ -648,7 +649,7 @@ public Task scrollAcrossLineage( entityList, input, maxHops, - filter, + validateAndConvert(filter), sortCriterionList, scrollId, keepAlive, @@ -683,10 +684,11 @@ public Task list( List sortCriterionList = getSortCriteria(sortCriteria, sortCriterion); - log.info("GET LIST RESULTS for {} with filter {}", entityName, filter); + final Filter finalFilter = validateAndConvert(filter); + log.info("GET LIST RESULTS for {} with filter {}", entityName, finalFilter); return RestliUtils.toTask( () -> { - SearchResult result = entitySearchService.filter(opContext, entityName, filter, sortCriterionList, start, count); + SearchResult result = entitySearchService.filter(opContext, entityName, finalFilter, sortCriterionList, start, count); if (!AuthUtil.isAPIAuthorizedResult( opContext, result)) { diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java index e40c4af1e0ae73..f8e138487fc168 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/CriterionUtils.java @@ -1,17 +1,81 @@ package com.linkedin.metadata.utils; +import static com.linkedin.metadata.Constants.URN_LI_PREFIX; + import com.linkedin.data.template.StringArray; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.Filter; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class CriterionUtils { private CriterionUtils() {} + /** + * This function is meant to validate and correct Filter input for rest.li endpoints. + * + * @param inputFilter the rest.li filter parameter + * @return validated and corrected filter + */ + @Nullable + public static Filter validateAndConvert(@Nullable Filter inputFilter) { + if (inputFilter != null) { + List invalidCriterion = new ArrayList<>(); + if (inputFilter.hasCriteria()) { + invalidCriterion.addAll( + inputFilter.getCriteria().stream() + .filter( + criterion -> + (criterion.hasValue() && !criterion.getValue().isEmpty()) + || !criterion.hasValue()) + .collect(Collectors.toList())); + } + if (inputFilter.hasOr()) { + invalidCriterion.addAll( + inputFilter.getOr().stream() + .flatMap(c -> c.getAnd().stream()) + .filter( + criterion -> + (criterion.hasValue() && !criterion.getValue().isEmpty()) + || !criterion.hasValue()) + .collect(Collectors.toList())); + } + + for (Criterion criterion : invalidCriterion) { + if (criterion.hasValue()) { + if ((criterion.getValue().contains(",") + && !criterion.getValue().startsWith(URN_LI_PREFIX)) + || criterion.getValue().contains(")," + URN_LI_PREFIX)) { + throw new IllegalArgumentException( + "Criterion `value` is deprecated and contains an ambiguous comma. Please use `values`."); + } + if (criterion.hasValues() && !criterion.getValue().equals(criterion.getValues().get(0))) { + throw new IllegalArgumentException( + "Criterion `value` is deprecated and `values`[0] is populated with a conflicting value."); + } + // auto-convert + if (!criterion.hasValues()) { + log.error( + "Deprecated use of a filter using Criterion's `value` has been detected and corrected. Please migrate to `values` instead."); + criterion.setValues(new StringArray(criterion.getValue())); + } + } + // must be set per required field + criterion.setValue(""); + } + } + return inputFilter; + } + public static Criterion buildExistsCriterion(@Nonnull String field) { return buildCriterion(field, Condition.EXISTS, false, Collections.emptyList()); } diff --git a/metadata-utils/src/test/java/com/linkedin/metadata/utils/CriterionUtilsTest.java b/metadata-utils/src/test/java/com/linkedin/metadata/utils/CriterionUtilsTest.java new file mode 100644 index 00000000000000..e2f22dd665c7c7 --- /dev/null +++ b/metadata-utils/src/test/java/com/linkedin/metadata/utils/CriterionUtilsTest.java @@ -0,0 +1,274 @@ +package com.linkedin.metadata.utils; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +import com.linkedin.data.template.StringArray; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import org.testng.annotations.Test; + +public class CriterionUtilsTest { + @Test + public void testNullFilter() { + Filter result = CriterionUtils.validateAndConvert(null); + assertNull(result); + } + + @Test + public void testEmptyFilter() { + Filter input = new Filter(); + Filter result = CriterionUtils.validateAndConvert(input); + assertNotNull(result); + assertFalse(result.hasCriteria()); + assertFalse(result.hasOr()); + } + + @Test + public void testSimpleCriterionConversion() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValue("testValue"); + input.setCriteria(new CriterionArray(criterion)); + + Filter result = CriterionUtils.validateAndConvert(input); + + Criterion convertedCriterion = result.getCriteria().get(0); + assertEquals(convertedCriterion.getValue(), ""); + assertTrue(convertedCriterion.hasValues()); + assertEquals("testValue", convertedCriterion.getValues().get(0)); + } + + @Test + public void testOrClauseCriterionConversion() { + Filter input = new Filter(); + + // Create OR clause with AND criteria + Criterion criterion = new Criterion(); + criterion.setValue("orValue"); + + ConjunctiveCriterion conjunctive = new ConjunctiveCriterion(); + conjunctive.setAnd(new CriterionArray(criterion)); + + input.setOr(new ConjunctiveCriterionArray(conjunctive)); + + Filter result = CriterionUtils.validateAndConvert(input); + + Criterion convertedCriterion = result.getOr().get(0).getAnd().get(0); + assertEquals(convertedCriterion.getValue(), ""); + assertTrue(convertedCriterion.hasValues()); + assertEquals("orValue", convertedCriterion.getValues().get(0)); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testCommaInValueThrowsException() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValue("value1,value2"); + input.setCriteria(new CriterionArray(criterion)); + + CriterionUtils.validateAndConvert(input); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testConflictingValuesThrowsException() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValue("value1"); + criterion.setValues(new StringArray("differentValue")); + input.setCriteria(new CriterionArray(criterion)); + + CriterionUtils.validateAndConvert(input); + } + + @Test + public void testExistingValuesNotModified() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValue("value1"); + criterion.setValues(new StringArray("value1")); // Same value, should not throw exception + input.setCriteria(new CriterionArray(criterion)); + + Filter result = CriterionUtils.validateAndConvert(input); + + Criterion convertedCriterion = result.getCriteria().get(0); + assertEquals(convertedCriterion.getValue(), ""); + assertTrue(convertedCriterion.hasValues()); + assertEquals("value1", convertedCriterion.getValues().get(0)); + } + + @Test + public void testMultipleCriteriaConversion() { + Filter input = new Filter(); + + Criterion criterion1 = new Criterion(); + criterion1.setValue("value1"); + + Criterion criterion2 = new Criterion(); + criterion2.setValue("value2"); + + input.setCriteria(new CriterionArray(criterion1, criterion2)); + + Filter result = CriterionUtils.validateAndConvert(input); + + assertEquals(2, result.getCriteria().size()); + + for (Criterion c : result.getCriteria()) { + assertEquals(c.getValue(), ""); + assertTrue(c.hasValues()); + assertTrue(c.getValues().get(0).equals("value1") || c.getValues().get(0).equals("value2")); + } + } + + @Test + public void testMixedCriteriaAndOrClause() { + Filter input = new Filter(); + + // Add direct criteria + Criterion criterion1 = new Criterion(); + criterion1.setValue("directValue"); + input.setCriteria(new CriterionArray(criterion1)); + + // Add OR clause with AND criteria + Criterion criterion2 = new Criterion(); + criterion2.setValue("orValue"); + ConjunctiveCriterion conjunctive = new ConjunctiveCriterion(); + conjunctive.setAnd(new CriterionArray(criterion2)); + input.setOr(new ConjunctiveCriterionArray(conjunctive)); + + Filter result = CriterionUtils.validateAndConvert(input); + + // Check direct criterion + Criterion convertedDirect = result.getCriteria().get(0); + assertEquals(convertedDirect.getValue(), ""); + assertTrue(convertedDirect.hasValues()); + assertEquals("directValue", convertedDirect.getValues().get(0)); + + // Check OR clause criterion + Criterion convertedOr = result.getOr().get(0).getAnd().get(0); + assertEquals(convertedOr.getValue(), ""); + assertTrue(convertedOr.hasValues()); + assertEquals("orValue", convertedOr.getValues().get(0)); + } + + @Test + public void testEmptyStringValueNotConverted() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValue(""); // Empty string value + input.setCriteria(new CriterionArray(criterion)); + + Filter result = CriterionUtils.validateAndConvert(input); + + Criterion convertedCriterion = result.getCriteria().get(0); + assertEquals(convertedCriterion.getValue(), ""); + assertFalse(convertedCriterion.hasValues()); // Should not be converted since value was empty + } + + @Test + public void testMixedEmptyAndNonEmptyValues() { + Filter input = new Filter(); + + Criterion emptyCriterion = new Criterion(); + emptyCriterion.setValue(""); + + Criterion nonEmptyCriterion = new Criterion(); + nonEmptyCriterion.setValue("value1"); + + input.setCriteria(new CriterionArray(emptyCriterion, nonEmptyCriterion)); + + Filter result = CriterionUtils.validateAndConvert(input); + + assertEquals(2, result.getCriteria().size()); + + // Check empty criterion + Criterion convertedEmpty = result.getCriteria().get(0); + assertEquals(convertedEmpty.getValue(), ""); + assertFalse(convertedEmpty.hasValues()); + + // Check non-empty criterion + Criterion convertedNonEmpty = result.getCriteria().get(1); + assertEquals(convertedNonEmpty.getValue(), ""); + assertTrue(convertedNonEmpty.hasValues()); + assertEquals(convertedNonEmpty.getValues().get(0), "value1"); + } + + @Test + public void testOrClauseWithEmptyValues() { + Filter input = new Filter(); + + // Create OR clause with mixed empty and non-empty criteria + Criterion emptyCriterion = new Criterion(); + emptyCriterion.setValue(""); + + Criterion nonEmptyCriterion = new Criterion(); + nonEmptyCriterion.setValue("orValue"); + + ConjunctiveCriterion conjunctive = new ConjunctiveCriterion(); + conjunctive.setAnd(new CriterionArray(emptyCriterion, nonEmptyCriterion)); + + input.setOr(new ConjunctiveCriterionArray(conjunctive)); + + Filter result = CriterionUtils.validateAndConvert(input); + + // Check empty criterion + Criterion convertedEmpty = result.getOr().get(0).getAnd().get(0); + assertEquals(convertedEmpty.getValue(), ""); + assertFalse(convertedEmpty.hasValues()); + + // Check non-empty criterion + Criterion convertedNonEmpty = result.getOr().get(0).getAnd().get(1); + assertEquals(convertedNonEmpty.getValue(), ""); + assertTrue(convertedNonEmpty.hasValues()); + assertEquals(convertedNonEmpty.getValues().get(0), "orValue"); + } + + @Test + public void testCriterionWithOnlyValues() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValues(new StringArray("value1")); // Only has values, no value field set + input.setCriteria(new CriterionArray(criterion)); + + Filter result = CriterionUtils.validateAndConvert(input); + + Criterion convertedCriterion = result.getCriteria().get(0); + assertEquals(convertedCriterion.getValue(), ""); + assertTrue(convertedCriterion.hasValues()); + assertEquals(convertedCriterion.getValues().get(0), "value1"); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testMultiUrnThrowsException() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValue( + "urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD),urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD)"); + input.setCriteria(new CriterionArray(criterion)); + + CriterionUtils.validateAndConvert(input); + } + + @Test + public void testUrnConversion() { + Filter input = new Filter(); + Criterion criterion = new Criterion(); + criterion.setValue("urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD)"); + input.setCriteria(new CriterionArray(criterion)); + + Filter result = CriterionUtils.validateAndConvert(input); + + Criterion convertedCriterion = result.getCriteria().get(0); + assertEquals(convertedCriterion.getValue(), ""); + assertTrue(convertedCriterion.hasValues()); + assertEquals( + "urn:li:dataset:(urn:li:dataPlatform:postgres,foo,PROD)", + convertedCriterion.getValues().get(0)); + } +} From 58068bfe2fa48849348f0e7c47ecf8ab148b1a75 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Sat, 23 Nov 2024 05:54:43 -0600 Subject: [PATCH 057/174] fix(validation): improved urn validation logic (#11935) --- docker/profiles/docker-compose.gms.yml | 2 + docs/deploy/environment-vars.md | 13 +-- docs/what/urn.md | 3 +- metadata-io/build.gradle | 2 + .../entity/validation/ValidationApiUtils.java | 54 +++++++++++ .../validation/ValidationApiUtilsTest.java | 95 +++++++++++++++++++ .../metadata/entity/EntityServiceTest.java | 2 +- 7 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml index 824c8024b05d63..e8b2d4cd1f29d3 100644 --- a/docker/profiles/docker-compose.gms.yml +++ b/docker/profiles/docker-compose.gms.yml @@ -101,6 +101,7 @@ x-datahub-gms-service: &datahub-gms-service <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env] ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-search_config.yaml} ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true} + STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true} healthcheck: test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health start_period: 90s @@ -183,6 +184,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service environment: &datahub-mce-consumer-env <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env] ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true} + STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true} x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev <<: *datahub-mce-consumer-service diff --git a/docs/deploy/environment-vars.md b/docs/deploy/environment-vars.md index 6429996c088b4a..13d7410397533d 100644 --- a/docs/deploy/environment-vars.md +++ b/docs/deploy/environment-vars.md @@ -9,12 +9,13 @@ DataHub works. ## Feature Flags -| Variable | Default | Unit/Type | Components | Description | -|--------------------------------------------------|---------|-----------|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| -| `UI_INGESTION_ENABLED` | `true` | boolean | [`GMS`, `MCE Consumer`] | Enable UI based ingestion. | -| `DATAHUB_ANALYTICS_ENABLED` | `true` | boolean | [`Frontend`, `GMS`] | Collect DataHub usage to populate the analytics dashboard. | -| `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true` | boolean | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Do not wait for the `system-update` to complete before starting. This should typically only be disabled during development. | -| `ER_MODEL_RELATIONSHIP_FEATURE_ENABLED` | `false` | boolean | [`Frontend`, `GMS`] | Enable ER Model Relation Feature that shows Relationships Tab within a Dataset UI. | +| Variable | Default | Unit/Type | Components | Description | +|--------------------------------------------------|----------|-----------|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `UI_INGESTION_ENABLED` | `true` | boolean | [`GMS`, `MCE Consumer`] | Enable UI based ingestion. | +| `DATAHUB_ANALYTICS_ENABLED` | `true` | boolean | [`Frontend`, `GMS`] | Collect DataHub usage to populate the analytics dashboard. | +| `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true` | boolean | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Do not wait for the `system-update` to complete before starting. This should typically only be disabled during development. | +| `ER_MODEL_RELATIONSHIP_FEATURE_ENABLED` | `false` | boolean | [`Frontend`, `GMS`] | Enable ER Model Relation Feature that shows Relationships Tab within a Dataset UI. | +| `STRICT_URN_VALIDATION_ENABLED` | `false` | boolean | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Enable stricter URN validation logic | ## Ingestion diff --git a/docs/what/urn.md b/docs/what/urn.md index e35ca7fbaca4bc..2f4dffb985653c 100644 --- a/docs/what/urn.md +++ b/docs/what/urn.md @@ -38,7 +38,8 @@ urn:li:dataset:(urn:li:dataPlatform:hdfs,PageViewEvent,EI) There are a few restrictions when creating an urn: 1. Commas are reserved character in URN fields: `,` -2. Parentheses are reserved characters in URN fields: `( , )` +2. Parentheses are reserved characters in URN fields: `(` or `)` 3. Colons are reserved characters in URN fields: `:` +4. Urn separator UTF-8 character `␟` Please do not use these characters when creating or generating urns. One approach is to use URL encoding for the characters. diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 09a41d100199d4..41294fab7b24a9 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -140,6 +140,8 @@ test { } testLogging.showStandardStreams = true testLogging.exceptionFormat = 'full' + + environment 'STRICT_URN_VALIDATION_ENABLED', 'true' } ebean { diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java index f7e639ecf36038..adc539cd926d32 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.entity.validation; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.validation.ValidationResult; import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.Constants; @@ -10,16 +11,26 @@ import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; import java.net.URISyntaxException; +import java.net.URLDecoder; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; @Slf4j public class ValidationApiUtils { + public static final String STRICT_URN_VALIDATION_ENABLED = "STRICT_URN_VALIDATION_ENABLED"; public static final int URN_NUM_BYTES_LIMIT = 512; + // Related to BrowsePathv2 public static final String URN_DELIMITER_SEPARATOR = "␟"; + // https://datahubproject.io/docs/what/urn/#restrictions + public static final Set ILLEGAL_URN_COMPONENT_CHARACTERS = Set.of(":", "(", ")", ","); /** * Validates a {@link RecordTemplate} and throws {@link ValidationException} if validation fails. @@ -38,6 +49,16 @@ public static void validateOrThrow(RecordTemplate record) { } public static void validateUrn(@Nonnull EntityRegistry entityRegistry, @Nonnull final Urn urn) { + validateUrn( + entityRegistry, + urn, + Boolean.TRUE.equals( + Boolean.parseBoolean( + System.getenv().getOrDefault(STRICT_URN_VALIDATION_ENABLED, "false")))); + } + + public static void validateUrn( + @Nonnull EntityRegistry entityRegistry, @Nonnull final Urn urn, boolean strict) { EntityRegistryUrnValidator validator = new EntityRegistryUrnValidator(entityRegistry); validator.setCurrentEntitySpec(entityRegistry.getEntitySpec(urn.getEntityType())); RecordTemplateValidator.validate( @@ -59,10 +80,32 @@ public static void validateUrn(@Nonnull EntityRegistry entityRegistry, @Nonnull + Integer.toString(URN_NUM_BYTES_LIMIT) + " bytes (when URL encoded)"); } + if (urn.toString().contains(URN_DELIMITER_SEPARATOR)) { throw new IllegalArgumentException( "Error: URN cannot contain " + URN_DELIMITER_SEPARATOR + " character"); } + + List illegalComponents = + urn.getEntityKey().getParts().stream() + .flatMap(ValidationApiUtils::processUrnPartRecursively) + .filter( + urnPart -> ILLEGAL_URN_COMPONENT_CHARACTERS.stream().anyMatch(urnPart::contains)) + .collect(Collectors.toList()); + + if (!illegalComponents.isEmpty()) { + String message = + String.format( + "Illegal `%s` characters detected in URN %s component(s): %s", + ILLEGAL_URN_COMPONENT_CHARACTERS, urn, illegalComponents); + + if (strict) { + throw new IllegalArgumentException(message); + } else { + log.error(message); + } + } + try { Urn.createFromString(urn.toString()); } catch (URISyntaxException e) { @@ -70,6 +113,17 @@ public static void validateUrn(@Nonnull EntityRegistry entityRegistry, @Nonnull } } + /** Recursively process URN parts with URL decoding */ + private static Stream processUrnPartRecursively(String urnPart) { + String decodedPart = URLDecoder.decode(urnPart, StandardCharsets.UTF_8); + if (decodedPart.startsWith("urn:li:")) { + // Recursively process nested URN after decoding + return UrnUtils.getUrn(decodedPart).getEntityKey().getParts().stream() + .flatMap(ValidationApiUtils::processUrnPartRecursively); + } + return Stream.of(decodedPart); + } + /** * Validates a {@link RecordTemplate} and logs a warning if validation fails. * diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java new file mode 100644 index 00000000000000..7a671a0a496309 --- /dev/null +++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java @@ -0,0 +1,95 @@ +package com.linkedin.metadata.entity.validation; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.models.registry.EntityRegistry; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.net.URISyntaxException; +import org.testng.annotations.Test; + +public class ValidationApiUtilsTest { + private static final EntityRegistry entityRegistry = + TestOperationContexts.defaultEntityRegistry(); + + @Test + public void testValidateDatasetUrn() { + Urn validUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,/path/to/data,PROD)"); + ValidationApiUtils.validateUrn(entityRegistry, validUrn, true); + // If no exception is thrown, test passes + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testSimpleUrnColon() { + Urn invalidUrn = UrnUtils.getUrn("urn:li:corpuser:foo:bar"); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + + @Test + public void testComplexUrnColon() throws URISyntaxException { + Urn validUrn = + Urn.createFromString( + "urn:li:dataset:(urn:li:dataPlatform:s3,urn:li:dataset:%28urn:li:dataPlatform:s3%2Ctest-datalake-concepts/prog_maintenance%2CPROD%29,PROD)"); + ValidationApiUtils.validateUrn(entityRegistry, validUrn, true); + // If no exception is thrown, test passes + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testUrnFabricType() { + Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,/path/to/data,())"); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testUrnWithTrailingWhitespace() { + Urn invalidUrn = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,/path/to/data,PROD) "); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testUrnWithIllegalDelimiter() { + Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs␟path,PROD)"); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testComplexUrnWithParens() { + Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,(illegal),PROD)"); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testSimpleUrnWithParens() { + Urn invalidUrn = UrnUtils.getUrn("urn:li:corpuser:(foo)123"); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testExcessiveLength() { + StringBuilder longPath = new StringBuilder("urn:li:dataset:(urn:li:dataPlatform:hdfs,"); + // Create a path that will exceed 512 bytes when URL encoded + for (int i = 0; i < 500; i++) { + longPath.append("very/long/path/"); + } + longPath.append(",PROD)"); + Urn invalidUrn = UrnUtils.getUrn(longPath.toString()); + + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + + @Test + public void testValidComplexUrn() { + Urn validUrn = + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:bigquery,myproject.dataset.table,PROD)"); + + ValidationApiUtils.validateUrn(entityRegistry, validUrn); + // If no exception is thrown, test passes + } + + @Test(expectedExceptions = NullPointerException.class) + public void testUrnNull() { + // Act + ValidationApiUtils.validateUrn(entityRegistry, null); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index d704037b39911e..654c448fdec946 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -1836,7 +1836,7 @@ public void testValidateUrn() throws Exception { ValidationApiUtils.validateUrn(opContext.getEntityRegistry(), urnWithMismatchedParens); Assert.fail("Should have raised IllegalArgumentException for URN with mismatched parens"); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("mismatched paren nesting")); + assertTrue(e.getMessage().contains("[test(Key]")); } Urn invalidType = new Urn("li", "fakeMadeUpType", new TupleKey("testKey")); From 766d36d1647951032006ba7dd5929c9ac514a409 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Sat, 23 Nov 2024 10:47:54 -0600 Subject: [PATCH 058/174] feat(search): adjust schema field boost (#11933) --- .../src/main/pegasus/com/linkedin/schema/SchemaField.pdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl index 0b72d376b0be49..61731e8d37fd69 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl @@ -16,7 +16,7 @@ record SchemaField { @Searchable = { "fieldName": "fieldPaths", "fieldType": "TEXT", - "boostScore": 5.0, + "boostScore": 1.0, "queryByDefault": "true" } fieldPath: SchemaFieldPath From dd892dfbb17b75f479a8bcf6e77f39b6a641e671 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:31:17 +0530 Subject: [PATCH 059/174] fix(ingest/bigquery): ignore include constraints for biglake datasets (#11874) --- .../source/bigquery_v2/bigquery_schema.py | 47 ++++++++++++++----- .../source/bigquery_v2/bigquery_schema_gen.py | 23 ++++----- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index 58317b108bef4c..be85d037af9cd1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -152,6 +152,21 @@ class BigqueryDataset: snapshots: List[BigqueryTableSnapshot] = field(default_factory=list) columns: List[BigqueryColumn] = field(default_factory=list) + # Some INFORMATION_SCHEMA views are not available for BigLake tables + # based on Amazon S3 and Blob Storage data. + # https://cloud.google.com/bigquery/docs/omni-introduction#limitations + # Omni Locations - https://cloud.google.com/bigquery/docs/omni-introduction#locations + def is_biglake_dataset(self) -> bool: + return self.location is not None and self.location.lower().startswith( + ("aws-", "azure-") + ) + + def supports_table_constraints(self) -> bool: + return not self.is_biglake_dataset() + + def supports_table_partitions(self) -> bool: + return not self.is_biglake_dataset() + @dataclass class BigqueryProject: @@ -541,18 +556,26 @@ def get_table_constraints_for_dataset( table_name=constraint.table_name, type=constraint.constraint_type, field_path=constraint.column_name, - referenced_project_id=constraint.referenced_catalog - if constraint.constraint_type == "FOREIGN KEY" - else None, - referenced_dataset=constraint.referenced_schema - if constraint.constraint_type == "FOREIGN KEY" - else None, - referenced_table_name=constraint.referenced_table - if constraint.constraint_type == "FOREIGN KEY" - else None, - referenced_column_name=constraint.referenced_column - if constraint.constraint_type == "FOREIGN KEY" - else None, + referenced_project_id=( + constraint.referenced_catalog + if constraint.constraint_type == "FOREIGN KEY" + else None + ), + referenced_dataset=( + constraint.referenced_schema + if constraint.constraint_type == "FOREIGN KEY" + else None + ), + referenced_table_name=( + constraint.referenced_table + if constraint.constraint_type == "FOREIGN KEY" + else None + ), + referenced_column_name=( + constraint.referenced_column + if constraint.constraint_type == "FOREIGN KEY" + else None + ), ) ) self.report.num_get_table_constraints_for_dataset_api_requests += 1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py index 6f3008ccfd6923..788016103d0fcd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py @@ -498,7 +498,10 @@ def _process_schema( report=self.report, rate_limiter=rate_limiter, ) - if self.config.include_table_constraints: + if ( + self.config.include_table_constraints + and bigquery_dataset.supports_table_constraints() + ): constraints = self.schema_api.get_table_constraints_for_dataset( project_id=project_id, dataset_name=dataset_name, report=self.report ) @@ -1157,9 +1160,11 @@ def gen_schema_metadata( # fields=[], fields=self.gen_schema_fields( columns, - table.constraints - if (isinstance(table, BigqueryTable) and table.constraints) - else [], + ( + table.constraints + if (isinstance(table, BigqueryTable) and table.constraints) + else [] + ), ), foreignKeys=foreign_keys if foreign_keys else None, ) @@ -1180,13 +1185,9 @@ def get_tables_for_dataset( ) -> Iterable[BigqueryTable]: # In bigquery there is no way to query all tables in a Project id with PerfTimer() as timer: - # PARTITIONS INFORMATION_SCHEMA view is not available for BigLake tables - # based on Amazon S3 and Blob Storage data. - # https://cloud.google.com/bigquery/docs/omni-introduction#limitations - # Omni Locations - https://cloud.google.com/bigquery/docs/omni-introduction#locations - with_partitions = self.config.have_table_data_read_permission and not ( - dataset.location - and dataset.location.lower().startswith(("aws-", "azure-")) + with_partitions = ( + self.config.have_table_data_read_permission + and dataset.supports_table_partitions() ) # Partitions view throw exception if we try to query partition info for too many tables From b5fb691f0da2293fdd3f6ee266e3a98f348cc6af Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:31:35 +0530 Subject: [PATCH 060/174] feat(ingest/kafka): improve error handling of oauth_cb config (#11929) --- .../configuration/kafka_consumer_config.py | 5 ++++- .../tests/unit/test_kafka_source.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py index d3ff5998d3e790..cac6bb4996391f 100644 --- a/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py +++ b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py @@ -30,6 +30,9 @@ def _resolve_oauth_callback(self) -> None: call_back = self.get_call_back_attribute() - assert call_back # to silent lint + assert isinstance(call_back, str), ( + "oauth_cb must be a string representing python function reference " + "in the format :." + ) # Set the callback self._config[CallableConsumerConfig.CALLBACK_ATTRIBUTE] = import_path(call_back) diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py index dfd32085b77055..cab0a2bce7ba8c 100644 --- a/metadata-ingestion/tests/unit/test_kafka_source.py +++ b/metadata-ingestion/tests/unit/test_kafka_source.py @@ -10,6 +10,7 @@ ) from freezegun import freeze_time +from datahub.configuration.common import ConfigurationError from datahub.emitter.mce_builder import ( OwnerType, make_dataplatform_instance_urn, @@ -738,3 +739,23 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: assert workunits[7].metadata.aspectName == "glossaryTermKey" assert workunits[8].metadata.aspectName == "tagKey" assert workunits[9].metadata.aspectName == "tagKey" + + +def test_kafka_source_oauth_cb_configuration(): + with pytest.raises( + ConfigurationError, + match=( + "oauth_cb must be a string representing python function reference " + "in the format :." + ), + ): + KafkaSourceConfig.parse_obj( + { + "connection": { + "bootstrap": "foobar:9092", + "consumer_config": { + "oauth_cb": test_kafka_ignore_warnings_on_schema_type + }, + } + } + ) From a1c71a48f2d1ab04f4fa9808045093281841aa22 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 25 Nov 2024 10:36:58 +0530 Subject: [PATCH 061/174] feat(ingest/oracle): support profile limits for large tables (#11827) --- .../source/bigquery_v2/bigquery_report.py | 4 +- .../source/dremio/dremio_reporting.py | 4 +- .../ingestion/source/ge_data_profiler.py | 2 +- .../ingestion/source/ge_profiling_config.py | 8 +- .../ingestion/source/redshift/report.py | 4 +- .../source/snowflake/snowflake_report.py | 4 +- .../datahub/ingestion/source/sql/oracle.py | 50 +++++++++++++ .../ingestion/source/sql/sql_common.py | 70 +++++------------ .../source/sql/sql_generic_profiler.py | 35 +-------- .../ingestion/source/sql/sql_report.py | 75 +++++++++++++++++++ .../datahub/ingestion/source/sql/teradata.py | 4 +- .../datahub/ingestion/source/sql/vertica.py | 4 +- .../datahub/ingestion/source/unity/report.py | 4 +- 13 files changed, 167 insertions(+), 101 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py index 7e8b2931282fff..06842da67f76ca 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_report.py @@ -8,7 +8,7 @@ from datahub.ingestion.api.report import Report from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin -from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport from datahub.ingestion.source_report.time_window import BaseTimeWindowReport from datahub.sql_parsing.sql_parsing_aggregator import SqlAggregatorReport @@ -77,7 +77,7 @@ class BigQueryQueriesExtractorReport(Report): @dataclass class BigQueryV2Report( - ProfilingSqlReport, + SQLSourceReport, IngestionStageReport, BaseTimeWindowReport, ClassificationReportMixin, diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py index ccc685382f374d..926dbd42eb2673 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from datetime import datetime -from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalSourceReport, ) @@ -10,7 +10,7 @@ @dataclass class DremioSourceReport( - ProfilingSqlReport, StaleEntityRemovalSourceReport, IngestionStageReport + SQLSourceReport, StaleEntityRemovalSourceReport, IngestionStageReport ): num_containers_failed: int = 0 num_datasets_failed: int = 0 diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index d175fce04a52c2..0d9dadaf6af555 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -55,7 +55,7 @@ Cardinality, convert_to_cardinality, ) -from datahub.ingestion.source.sql.sql_common import SQLSourceReport +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.metadata.com.linkedin.pegasus2avro.schema import EditableSchemaMetadata from datahub.metadata.schema_classes import ( DatasetFieldProfileClass, diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index c20506e36a844f..9bf4451a188bcb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -125,12 +125,16 @@ class GEProfilingConfig(GEProfilingBaseConfig): profile_table_size_limit: Optional[int] = Field( default=5, - description="Profile tables only if their size is less then specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `snowflake` and `BigQuery`", + description="Profile tables only if their size is less than specified GBs. If set to `null`, " + "no limit on the size of tables to profile. Supported only in `snowflake` and `BigQuery`" + "Supported for `oracle` based on calculated size from gathered stats.", ) profile_table_row_limit: Optional[int] = Field( default=5000000, - description="Profile tables only if their row count is less then specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `snowflake` and `BigQuery`", + description="Profile tables only if their row count is less than specified count. If set to `null`, " + "no limit on the row count of tables to profile. Supported only in `snowflake` and `BigQuery`" + "Supported for `oracle` based on gathered stats.", ) profile_table_row_count_estimate_only: bool = Field( diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py index ff28ed2c5e849c..2748f2a588a930 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/report.py @@ -3,7 +3,7 @@ from typing import Dict, Optional from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin -from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport from datahub.ingestion.source_report.time_window import BaseTimeWindowReport from datahub.sql_parsing.sql_parsing_aggregator import SqlAggregatorReport @@ -14,7 +14,7 @@ @dataclass class RedshiftReport( - ProfilingSqlReport, + SQLSourceReport, IngestionStageReport, BaseTimeWindowReport, ClassificationReportMixin, diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index 80b6be36e5ffa1..b5f56f99431f91 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -5,7 +5,7 @@ from datahub.ingestion.api.report import Report from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin from datahub.ingestion.source.snowflake.constants import SnowflakeEdition -from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionReport, ) @@ -59,7 +59,7 @@ class SnowflakeUsageReport: @dataclass -class SnowflakeReport(ProfilingSqlReport, BaseTimeWindowReport): +class SnowflakeReport(SQLSourceReport, BaseTimeWindowReport): num_table_to_table_edges_scanned: int = 0 num_table_to_view_edges_scanned: int = 0 num_view_to_table_edges_scanned: int = 0 diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py index 766b704d6ffafe..52db3cd11a759d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/oracle.py @@ -1,3 +1,4 @@ +import datetime import logging import re @@ -631,3 +632,52 @@ def get_workunits(self): clear=False, ): return super().get_workunits() + + def generate_profile_candidates( + self, + inspector: Inspector, + threshold_time: Optional[datetime.datetime], + schema: str, + ) -> Optional[List[str]]: + tables_table_name = ( + "ALL_TABLES" if self.config.data_dictionary_mode == "ALL" else "DBA_TABLES" + ) + + # If stats are available , they are used even if they are stale. + # Assuming that the table would typically grow over time, this will ensure to filter + # large tables known at stats collection time from profiling candidates. + # If stats are not available (NULL), such tables are not filtered and are considered + # as profiling candidates. + cursor = inspector.bind.execute( + sql.text( + f"""SELECT + t.OWNER, + t.TABLE_NAME, + t.NUM_ROWS, + t.LAST_ANALYZED, + COALESCE(t.NUM_ROWS * t.AVG_ROW_LEN, 0) / (1024 * 1024 * 1024) AS SIZE_GB + FROM {tables_table_name} t + WHERE t.OWNER = :owner + AND (t.NUM_ROWS < :table_row_limit OR t.NUM_ROWS IS NULL) + AND COALESCE(t.NUM_ROWS * t.AVG_ROW_LEN, 0) / (1024 * 1024 * 1024) < :table_size_limit + """ + ), + dict( + owner=inspector.dialect.denormalize_name(schema), + table_row_limit=self.config.profiling.profile_table_row_limit, + table_size_limit=self.config.profiling.profile_table_size_limit, + ), + ) + + TABLE_NAME_COL_LOC = 1 + return [ + self.get_identifier( + schema=schema, + entity=inspector.dialect.normalize_name(row[TABLE_NAME_COL_LOC]) + or _raise_err( + ValueError(f"Invalid table name: {row[TABLE_NAME_COL_LOC]}") + ), + inspector=inspector, + ) + for row in cursor + ] diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index ae6116326da33c..41ffcb95a7cc43 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -51,7 +51,6 @@ from datahub.ingestion.glossary.classification_mixin import ( SAMPLE_SIZE_MULTIPLIER, ClassificationHandler, - ClassificationReportMixin, ) from datahub.ingestion.source.common.data_reader import DataReader from datahub.ingestion.source.common.subtypes import ( @@ -59,6 +58,7 @@ DatasetSubTypes, ) from datahub.ingestion.source.sql.sql_config import SQLCommonConfig +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source.sql.sql_utils import ( add_table_to_schema_container, downgrade_schema_from_v2, @@ -74,7 +74,6 @@ ) from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, - StaleEntityRemovalSourceReport, ) from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionSourceBase, @@ -118,9 +117,7 @@ ) from datahub.telemetry import telemetry from datahub.utilities.file_backed_collections import FileBackedDict -from datahub.utilities.lossy_collections import LossyList from datahub.utilities.registries.domain_registry import DomainRegistry -from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport from datahub.utilities.sqlalchemy_type_converter import ( get_native_data_type_for_sqlalchemy_type, ) @@ -134,43 +131,6 @@ logger: logging.Logger = logging.getLogger(__name__) -@dataclass -class SQLSourceReport(StaleEntityRemovalSourceReport, ClassificationReportMixin): - tables_scanned: int = 0 - views_scanned: int = 0 - entities_profiled: int = 0 - filtered: LossyList[str] = field(default_factory=LossyList) - - query_combiner: Optional[SQLAlchemyQueryCombinerReport] = None - - num_view_definitions_parsed: int = 0 - num_view_definitions_failed_parsing: int = 0 - num_view_definitions_failed_column_parsing: int = 0 - view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList) - - def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: - """ - Entity could be a view or a table - """ - if ent_type == "table": - self.tables_scanned += 1 - elif ent_type == "view": - self.views_scanned += 1 - else: - raise KeyError(f"Unknown entity {ent_type}.") - - def report_entity_profiled(self, name: str) -> None: - self.entities_profiled += 1 - - def report_dropped(self, ent_name: str) -> None: - self.filtered.append(ent_name) - - def report_from_query_combiner( - self, query_combiner_report: SQLAlchemyQueryCombinerReport - ) -> None: - self.query_combiner = query_combiner_report - - class SqlWorkUnit(MetadataWorkUnit): pass @@ -352,7 +312,7 @@ class SQLAlchemySource(StatefulIngestionSourceBase, TestableSource): def __init__(self, config: SQLCommonConfig, ctx: PipelineContext, platform: str): super().__init__(config, ctx) - self.config = config + self.config: SQLCommonConfig = config self.platform = platform self.report: SQLSourceReport = SQLSourceReport() self.profile_metadata_info: ProfileMetadata = ProfileMetadata() @@ -1282,17 +1242,22 @@ def generate_profile_candidates( def is_dataset_eligible_for_profiling( self, dataset_name: str, - sql_config: SQLCommonConfig, + schema: str, inspector: Inspector, profile_candidates: Optional[List[str]], ) -> bool: - return ( - sql_config.table_pattern.allowed(dataset_name) - and sql_config.profile_pattern.allowed(dataset_name) - ) and ( - profile_candidates is None - or (profile_candidates is not None and dataset_name in profile_candidates) - ) + if not ( + self.config.table_pattern.allowed(dataset_name) + and self.config.profile_pattern.allowed(dataset_name) + ): + self.report.profiling_skipped_table_profile_pattern[schema] += 1 + return False + + if profile_candidates is not None and dataset_name not in profile_candidates: + self.report.profiling_skipped_other[schema] += 1 + return False + + return True def loop_profiler_requests( self, @@ -1307,7 +1272,7 @@ def loop_profiler_requests( if ( sql_config.profiling.profile_if_updated_since_days is not None or sql_config.profiling.profile_table_size_limit is not None - or sql_config.profiling.profile_table_row_limit is None + or sql_config.profiling.profile_table_row_limit is not None ): try: threshold_time: Optional[datetime.datetime] = None @@ -1328,8 +1293,9 @@ def loop_profiler_requests( schema=schema, entity=table, inspector=inspector ) if not self.is_dataset_eligible_for_profiling( - dataset_name, sql_config, inspector, profile_candidates + dataset_name, schema, inspector, profile_candidates ): + self.report.num_tables_not_eligible_profiling[schema] += 1 if self.config.profiling.report_dropped_profiles: self.report.report_dropped(f"profile of {dataset_name}") continue diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py index 9c8e475e7b3074..bd6c23cc2d4644 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_generic_profiler.py @@ -1,6 +1,6 @@ import logging from abc import abstractmethod -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Dict, Iterable, List, Optional, Union, cast @@ -14,42 +14,13 @@ DatahubGEProfiler, GEProfilerRequest, ) -from datahub.ingestion.source.sql.sql_common import SQLSourceReport from datahub.ingestion.source.sql.sql_config import SQLCommonConfig from datahub.ingestion.source.sql.sql_generic import BaseTable, BaseView +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source.sql.sql_utils import check_table_with_profile_pattern from datahub.ingestion.source.state.profiling_state_handler import ProfilingHandler from datahub.metadata.com.linkedin.pegasus2avro.dataset import DatasetProfile from datahub.metadata.com.linkedin.pegasus2avro.timeseries import PartitionType -from datahub.utilities.stats_collections import TopKDict, int_top_k_dict - - -@dataclass -class DetailedProfilerReportMixin: - profiling_skipped_not_updated: TopKDict[str, int] = field( - default_factory=int_top_k_dict - ) - profiling_skipped_size_limit: TopKDict[str, int] = field( - default_factory=int_top_k_dict - ) - - profiling_skipped_row_limit: TopKDict[str, int] = field( - default_factory=int_top_k_dict - ) - - profiling_skipped_table_profile_pattern: TopKDict[str, int] = field( - default_factory=int_top_k_dict - ) - - profiling_skipped_other: TopKDict[str, int] = field(default_factory=int_top_k_dict) - - num_tables_not_eligible_profiling: Dict[str, int] = field( - default_factory=int_top_k_dict - ) - - -class ProfilingSqlReport(DetailedProfilerReportMixin, SQLSourceReport): - pass @dataclass @@ -65,7 +36,7 @@ class GenericProfiler: def __init__( self, config: SQLCommonConfig, - report: ProfilingSqlReport, + report: SQLSourceReport, platform: str, state_handler: Optional[ProfilingHandler] = None, ) -> None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py new file mode 100644 index 00000000000000..c1f722b5d1e783 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py @@ -0,0 +1,75 @@ +from dataclasses import dataclass, field +from typing import Dict, Optional + +from datahub.ingestion.glossary.classification_mixin import ClassificationReportMixin +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, +) +from datahub.utilities.lossy_collections import LossyList +from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport +from datahub.utilities.stats_collections import TopKDict, int_top_k_dict + + +@dataclass +class DetailedProfilerReportMixin: + profiling_skipped_not_updated: TopKDict[str, int] = field( + default_factory=int_top_k_dict + ) + profiling_skipped_size_limit: TopKDict[str, int] = field( + default_factory=int_top_k_dict + ) + + profiling_skipped_row_limit: TopKDict[str, int] = field( + default_factory=int_top_k_dict + ) + + profiling_skipped_table_profile_pattern: TopKDict[str, int] = field( + default_factory=int_top_k_dict + ) + + profiling_skipped_other: TopKDict[str, int] = field(default_factory=int_top_k_dict) + + num_tables_not_eligible_profiling: Dict[str, int] = field( + default_factory=int_top_k_dict + ) + + +@dataclass +class SQLSourceReport( + StaleEntityRemovalSourceReport, + ClassificationReportMixin, + DetailedProfilerReportMixin, +): + tables_scanned: int = 0 + views_scanned: int = 0 + entities_profiled: int = 0 + filtered: LossyList[str] = field(default_factory=LossyList) + + query_combiner: Optional[SQLAlchemyQueryCombinerReport] = None + + num_view_definitions_parsed: int = 0 + num_view_definitions_failed_parsing: int = 0 + num_view_definitions_failed_column_parsing: int = 0 + view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList) + + def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: + """ + Entity could be a view or a table + """ + if ent_type == "table": + self.tables_scanned += 1 + elif ent_type == "view": + self.views_scanned += 1 + else: + raise KeyError(f"Unknown entity {ent_type}.") + + def report_entity_profiled(self, name: str) -> None: + self.entities_profiled += 1 + + def report_dropped(self, ent_name: str) -> None: + self.filtered.append(ent_name) + + def report_from_query_combiner( + self, query_combiner_report: SQLAlchemyQueryCombinerReport + ) -> None: + self.query_combiner = query_combiner_report diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py index 53b1ddfcde5952..e42564975c3d19 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py @@ -44,7 +44,7 @@ from datahub.ingestion.graph.client import DataHubGraph from datahub.ingestion.source.sql.sql_common import SqlWorkUnit, register_custom_type from datahub.ingestion.source.sql.sql_config import SQLCommonConfig -from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source.sql.two_tier_sql_source import ( TwoTierSQLAlchemyConfig, TwoTierSQLAlchemySource, @@ -330,7 +330,7 @@ def optimized_get_view_definition( @dataclass -class TeradataReport(ProfilingSqlReport, IngestionStageReport, BaseTimeWindowReport): +class TeradataReport(SQLSourceReport, IngestionStageReport, BaseTimeWindowReport): num_queries_parsed: int = 0 num_view_ddl_parsed: int = 0 num_table_parse_failures: int = 0 diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index a340f049731c46..92487d48b99e63 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -27,7 +27,6 @@ from datahub.ingestion.source.common.data_reader import DataReader from datahub.ingestion.source.sql.sql_common import ( SQLAlchemySource, - SQLSourceReport, SqlWorkUnit, get_schema_metadata, ) @@ -35,6 +34,7 @@ BasicSQLAlchemyConfig, SQLCommonConfig, ) +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source.sql.sql_utils import get_domain_wu from datahub.metadata.com.linkedin.pegasus2avro.common import StatusClass from datahub.metadata.com.linkedin.pegasus2avro.dataset import UpstreamLineage @@ -536,7 +536,7 @@ def loop_profiler_requests( ) if not self.is_dataset_eligible_for_profiling( - dataset_name, sql_config, inspector, profile_candidates + dataset_name, schema, inspector, profile_candidates ): if self.config.profiling.report_dropped_profiles: self.report.report_dropped(f"profile of {dataset_name}") diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py index f4579376a3b3a4..f16769341853a1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/report.py @@ -2,7 +2,7 @@ from typing import Optional, Tuple from datahub.ingestion.api.report import EntityFilterReport, Report -from datahub.ingestion.source.sql.sql_generic_profiler import ProfilingSqlReport +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport from datahub.utilities.lossy_collections import LossyDict, LossyList from datahub.utilities.perf_timer import PerfTimer @@ -19,7 +19,7 @@ class UnityCatalogUsagePerfReport(Report): @dataclass -class UnityCatalogReport(IngestionStageReport, ProfilingSqlReport): +class UnityCatalogReport(IngestionStageReport, SQLSourceReport): metastores: EntityFilterReport = EntityFilterReport.field(type="metastore") catalogs: EntityFilterReport = EntityFilterReport.field(type="catalog") schemas: EntityFilterReport = EntityFilterReport.field(type="schema") From 56a5dcaaea37dc50d1ee8ce4898dbc129d7e82f8 Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Mon, 25 Nov 2024 15:56:26 +0900 Subject: [PATCH 062/174] fix: remove ai summit post (#11826) --- .../examples/mce_files/bootstrap_mce.json | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/metadata-ingestion/examples/mce_files/bootstrap_mce.json b/metadata-ingestion/examples/mce_files/bootstrap_mce.json index f0c4e7ff996ed3..bc218e5e8c2d53 100644 --- a/metadata-ingestion/examples/mce_files/bootstrap_mce.json +++ b/metadata-ingestion/examples/mce_files/bootstrap_mce.json @@ -3613,33 +3613,6 @@ }, "systemMetadata": null }, - { - "entityType": "post", - "entityUrn": "urn:li:post:f3a68539-f7e4-4c41-a4fd-9e57c085d8de", - "changeType": "UPSERT", - "aspectName": "postInfo", - "aspect": { - "json": { - "type": "HOME_PAGE_ANNOUNCEMENT", - "content": { - "title": "Join Metadata & AI Summit 2024", - "type": "LINK", - "link": "http://www.acryldata.io/conference?utm_source=datahub_quickstart&utm_medium=metadata_ai_2024&utm_campaign=pinned_announcement", - "media": { - "type": "IMAGE", - "location": "https://formulatedby.com/wp-content/uploads/2024/07/0193320a6d93e7508d1598f7b24662f75a87e92f-352x456-1.svg" - } - }, - "created": 1712547125049, - "lastModified": 1712547125049 - } - }, - "systemMetadata": { - "lastObserved": 1712548844816, - "runId": "datahub-2024_04_08-13_00_44", - "lastRunId": "no-run-id-provided" - } - }, { "entityType": "post", "entityUrn": "urn:li:post:f3a68539-f7e4-4c41-a4fd-9e57c085d8dd", From 41f7fc9949967d0fc871adf875ba35c2def09856 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 25 Nov 2024 09:49:16 -0500 Subject: [PATCH 063/174] fix(ui/graphql) Handle groups in institutionalMemory aspect (#11934) --- .../datahub/graphql/GmsGraphQLEngine.java | 25 +++++++++++---- .../resolvers/type/ResolvedActorResolver.java | 25 +++++++++++++++ .../InstitutionalMemoryMetadataMapper.java | 1 + .../common/mappers/ResolvedActorMapper.java | 31 +++++++++++++++++++ .../src/main/resources/entity.graphql | 10 +++++- datahub-web-react/src/Mocks.tsx | 6 ++++ .../containers/profile/sidebar/LinkButton.tsx | 2 +- .../Documentation/components/LinkList.tsx | 8 ++--- .../src/graphql-mock/mutationHelper.ts | 1 + datahub-web-react/src/graphql/domain.graphql | 5 ++- .../src/graphql/fragments.graphql | 16 ++++++++-- 11 files changed, 111 insertions(+), 19 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 5b265b67144523..d1da55268a50d5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -63,6 +63,7 @@ import com.linkedin.datahub.graphql.generated.Domain; import com.linkedin.datahub.graphql.generated.ERModelRelationship; import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties; +import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityPath; import com.linkedin.datahub.graphql.generated.EntityRelationship; import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy; @@ -312,6 +313,7 @@ import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.PropertyValueResolver; +import com.linkedin.datahub.graphql.resolvers.type.ResolvedActorResolver; import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver; import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver; import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver; @@ -1730,12 +1732,22 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { .type( "InstitutionalMemoryMetadata", typeWiring -> - typeWiring.dataFetcher( - "author", - new LoadableTypeResolver<>( - corpUserType, - (env) -> - ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn()))) + typeWiring + .dataFetcher( + "author", + new LoadableTypeResolver<>( + corpUserType, + (env) -> + ((InstitutionalMemoryMetadata) env.getSource()) + .getAuthor() + .getUrn())) + .dataFetcher( + "actor", + new EntityTypeResolver( + this.entityTypes, + (env) -> + (Entity) + ((InstitutionalMemoryMetadata) env.getSource()).getActor()))) .type( "DatasetStatsSummary", typeWiring -> @@ -2242,6 +2254,7 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) { "HyperParameterValueType", typeWiring -> typeWiring.typeResolver(new HyperParameterValueTypeResolver())) .type("PropertyValue", typeWiring -> typeWiring.typeResolver(new PropertyValueResolver())) + .type("ResolvedActor", typeWiring -> typeWiring.typeResolver(new ResolvedActorResolver())) .type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver())) .type( "TimeSeriesAspect", diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java new file mode 100644 index 00000000000000..7ae719a23b00ad --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/type/ResolvedActorResolver.java @@ -0,0 +1,25 @@ +package com.linkedin.datahub.graphql.resolvers.type; + +import com.linkedin.datahub.graphql.generated.CorpGroup; +import com.linkedin.datahub.graphql.generated.CorpUser; +import graphql.TypeResolutionEnvironment; +import graphql.schema.GraphQLObjectType; +import graphql.schema.TypeResolver; + +public class ResolvedActorResolver implements TypeResolver { + + public static final String CORP_USER = "CorpUser"; + public static final String CORP_GROUP = "CorpGroup"; + + @Override + public GraphQLObjectType getType(TypeResolutionEnvironment env) { + if (env.getObject() instanceof CorpUser) { + return env.getSchema().getObjectType(CORP_USER); + } else if (env.getObject() instanceof CorpGroup) { + return env.getSchema().getObjectType(CORP_GROUP); + } else { + throw new RuntimeException( + "Unrecognized object type provided to type resolver, Type:" + env.getObject().toString()); + } + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java index 7c6de02ecc8767..9781643c414c81 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/InstitutionalMemoryMetadataMapper.java @@ -28,6 +28,7 @@ public InstitutionalMemoryMetadata apply( result.setDescription(input.getDescription()); // deprecated field result.setLabel(input.getDescription()); result.setAuthor(getAuthor(input.getCreateStamp().getActor().toString())); + result.setActor(ResolvedActorMapper.map(input.getCreateStamp().getActor())); result.setCreated(AuditStampMapper.map(context, input.getCreateStamp())); result.setAssociatedUrn(entityUrn.toString()); return result; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java new file mode 100644 index 00000000000000..c00ffd0b828b18 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/ResolvedActorMapper.java @@ -0,0 +1,31 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.generated.CorpGroup; +import com.linkedin.datahub.graphql.generated.CorpUser; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.ResolvedActor; +import com.linkedin.metadata.Constants; +import javax.annotation.Nonnull; + +public class ResolvedActorMapper { + + public static final ResolvedActorMapper INSTANCE = new ResolvedActorMapper(); + + public static ResolvedActor map(@Nonnull final Urn actorUrn) { + return INSTANCE.apply(actorUrn); + } + + public ResolvedActor apply(@Nonnull final Urn actorUrn) { + if (actorUrn.getEntityType().equals(Constants.CORP_GROUP_ENTITY_NAME)) { + CorpGroup partialGroup = new CorpGroup(); + partialGroup.setUrn(actorUrn.toString()); + partialGroup.setType(EntityType.CORP_GROUP); + return partialGroup; + } + CorpUser partialUser = new CorpUser(); + partialUser.setUrn(actorUrn.toString()); + partialUser.setType(EntityType.CORP_USER); + return (ResolvedActor) partialUser; + } +} diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 732a782139b616..049527e5d77e3b 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -3005,8 +3005,14 @@ type InstitutionalMemoryMetadata { """ The author of this metadata + Deprecated! Use actor instead for users or groups. """ - author: CorpUser! + author: CorpUser! @deprecated(reason: "Use `actor`") + + """ + The author of this metadata + """ + actor: ResolvedActor! """ An AuditStamp corresponding to the creation of this resource @@ -3834,6 +3840,8 @@ enum CorpUserStatus { ACTIVE } +union ResolvedActor = CorpUser | CorpGroup + """ A DataHub User entity, which represents a Person on the Metadata Entity Graph """ diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index aed672a34e7caf..329d6250e576ab 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -566,6 +566,12 @@ export const dataset3 = { username: 'datahub', type: EntityType.CorpUser, }, + actor: { + __typename: 'CorpUser', + urn: 'urn:li:corpuser:datahub', + username: 'datahub', + type: EntityType.CorpUser, + }, description: 'This only points to Google', label: 'This only points to Google', created: { diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx index 0ce3c9641d5597..c3896baedace79 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/LinkButton.tsx @@ -29,7 +29,7 @@ export default function LinkButton({ link }: Props) { href={link.url} target="_blank" rel="noreferrer" - key={`${link.label}-${link.url}-${link.author}`} + key={`${link.label}-${link.url}-${link.actor.urn}`} > {link.description || link.label} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx index 7212198bbf61ca..6eb680785599e1 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Documentation/components/LinkList.tsx @@ -3,7 +3,7 @@ import { Link } from 'react-router-dom'; import styled from 'styled-components/macro'; import { message, Button, List, Typography, Modal, Form, Input } from 'antd'; import { LinkOutlined, DeleteOutlined, EditOutlined } from '@ant-design/icons'; -import { EntityType, InstitutionalMemoryMetadata } from '../../../../../../types.generated'; +import { InstitutionalMemoryMetadata } from '../../../../../../types.generated'; import { useEntityData, useMutationUrn } from '../../../EntityContext'; import { useEntityRegistry } from '../../../../../useEntityRegistry'; import { ANTD_GRAY } from '../../../constants'; @@ -182,10 +182,8 @@ export const LinkList = ({ refetch }: LinkListProps) => { description={ <> Added {formatDateString(link.created.time)} by{' '} - - {link.author.username} + + {entityRegistry.getDisplayName(link.actor.type, link.actor)} } diff --git a/datahub-web-react/src/graphql-mock/mutationHelper.ts b/datahub-web-react/src/graphql-mock/mutationHelper.ts index a97b41b53bc656..0cf4f5f87f29ca 100644 --- a/datahub-web-react/src/graphql-mock/mutationHelper.ts +++ b/datahub-web-react/src/graphql-mock/mutationHelper.ts @@ -99,6 +99,7 @@ export const updateEntityLink = ({ entity, institutionalMemory }: UpdateEntityLi description: e.description as string, label: e.description as string, author: { urn: e.author, username: '', type: EntityType.CorpUser }, + actor: { urn: e.author, username: '', type: EntityType.CorpUser }, created: { time: Date.now(), actor: getActor(), __typename: 'AuditStamp' }, associatedUrn: dataEntity.urn, }; diff --git a/datahub-web-react/src/graphql/domain.graphql b/datahub-web-react/src/graphql/domain.graphql index 3897a2ced85b8f..2e96a78b0f44b4 100644 --- a/datahub-web-react/src/graphql/domain.graphql +++ b/datahub-web-react/src/graphql/domain.graphql @@ -19,9 +19,8 @@ query getDomain($urn: String!) { institutionalMemory { elements { url - author { - urn - username + actor { + ...resolvedActorFields } description created { diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index 7ce4082c42f61d..67dbdbbb22f309 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -202,12 +202,22 @@ fragment embedFields on Embed { renderUrl } +fragment resolvedActorFields on ResolvedActor { + ... on CorpUser { + urn + ...entityDisplayNameFields + } + ... on CorpGroup { + urn + ...entityDisplayNameFields + } +} + fragment institutionalMemoryFields on InstitutionalMemory { elements { url - author { - urn - username + actor { + ...resolvedActorFields } description created { From 5da39044e3350cc2da1258219b9a653b88174654 Mon Sep 17 00:00:00 2001 From: Pinaki Bhattacharjee Date: Mon, 25 Nov 2024 23:43:47 +0530 Subject: [PATCH 064/174] fix(browseDAO): Handle null browse path from ES in BrowseDAO (#11875) --- .../metadata/search/elasticsearch/query/ESBrowseDAO.java | 6 +++++- .../linkedin/metadata/search/query/BrowseDAOTest.java | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 61bba11098fae2..35f133cc794f2a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -36,6 +36,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -425,7 +426,10 @@ public List getBrowsePaths( if (!sourceMap.containsKey(BROWSE_PATH)) { return Collections.emptyList(); } - return (List) sourceMap.get(BROWSE_PATH); + List browsePaths = + ((List) sourceMap.get(BROWSE_PATH)) + .stream().filter(Objects::nonNull).collect(Collectors.toList()); + return browsePaths; } public BrowseResultV2 browseV2( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java index e71865921678bb..2ee96dd101da57 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/BrowseDAOTest.java @@ -102,5 +102,14 @@ public void testGetBrowsePath() throws Exception { List browsePaths = browseDAO.getBrowsePaths(opContext, "dataset", dummyUrn); assertEquals(browsePaths.size(), 1); assertEquals(browsePaths.get(0), "foo"); + + // Test the case of null browsePaths field + sourceMap.put("browsePaths", Collections.singletonList(null)); + when(mockSearchHit.getSourceAsMap()).thenReturn(sourceMap); + when(mockSearchHits.getHits()).thenReturn(new SearchHit[] {mockSearchHit}); + when(mockSearchResponse.getHits()).thenReturn(mockSearchHits); + when(mockClient.search(any(), eq(RequestOptions.DEFAULT))).thenReturn(mockSearchResponse); + List nullBrowsePaths = browseDAO.getBrowsePaths(opContext, "dataset", dummyUrn); + assertEquals(nullBrowsePaths.size(), 0); } } From 418259b8c4d48c9607efd4a3671fa08bde955816 Mon Sep 17 00:00:00 2001 From: JaeMoo Han Date: Tue, 26 Nov 2024 05:13:26 +0900 Subject: [PATCH 065/174] fix(airflow): remove trino to presto mapping (#11925) Co-authored-by: Harshal Sheth --- .../src/datahub_airflow_plugin/_datahub_ol_adapter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py index 7d35791bf1db42..69de61aced0a59 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_datahub_ol_adapter.py @@ -8,7 +8,6 @@ OL_SCHEME_TWEAKS = { "sqlserver": "mssql", - "trino": "presto", "awsathena": "athena", } From 90fe14aadbd12e457229e1927e8e213e8cc80959 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Tue, 26 Nov 2024 01:44:10 +0530 Subject: [PATCH 066/174] docs(ingest/bigquery): add partition support capability (#11940) --- .../src/datahub/ingestion/source/bigquery_v2/bigquery.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 76c2fbf48ccaba..16a5268a2dea76 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -95,6 +95,10 @@ def cleanup(config: BigQueryV2Config) -> None: "Optionally enabled via `classification.enabled`", supported=True, ) +@capability( + SourceCapability.PARTITION_SUPPORT, + "Enabled by default, partition keys and clustering keys are supported.", +) class BigqueryV2Source(StatefulIngestionSourceBase, TestableSource): def __init__(self, ctx: PipelineContext, config: BigQueryV2Config): super().__init__(config, ctx) From 32ef3894400ac3aaac6f9d5b780aa58a9f6fb48c Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Mon, 25 Nov 2024 21:20:02 -0600 Subject: [PATCH 067/174] fix(dataProduct): reduce write fan-out for unset side effect (#11951) --- .../DataProductUnsetSideEffect.java | 61 +++++----- .../DataProductUnsetSideEffectTest.java | 107 ++++++++++++++++++ 2 files changed, 141 insertions(+), 27 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java b/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java index 544040d14f8b7c..9c4bb52f014fc1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -70,6 +71,7 @@ private static Stream generatePatchRemove( log.error("Unable to process data product properties for urn: {}", mclItem.getUrn()); return Stream.empty(); } + Map> patchOpMap = new HashMap<>(); for (DataProductAssociation dataProductAssociation : Optional.ofNullable(dataProductProperties.getAssets()) .orElse(new DataProductAssociationArray())) { @@ -93,40 +95,45 @@ private static Stream generatePatchRemove( if (!result.getEntities().isEmpty()) { for (RelatedEntities entity : result.getEntities()) { if (!mclItem.getUrn().equals(UrnUtils.getUrn(entity.getSourceUrn()))) { - EntitySpec entitySpec = - retrieverContext - .getAspectRetriever() - .getEntityRegistry() - .getEntitySpec(DATA_PRODUCT_ENTITY_NAME); GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp(); patchOp.setOp(PatchOperationType.REMOVE.getValue()); patchOp.setPath(String.format("/assets/%s", entity.getDestinationUrn())); - mcpItems.add( - PatchItemImpl.builder() - .urn(UrnUtils.getUrn(entity.getSourceUrn())) - .entitySpec( - retrieverContext - .getAspectRetriever() - .getEntityRegistry() - .getEntitySpec(DATA_PRODUCT_ENTITY_NAME)) - .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME) - .aspectSpec(entitySpec.getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys( - Map.of( - DataProductPropertiesTemplate.ASSETS_FIELD_NAME, - List.of(DataProductPropertiesTemplate.KEY_FIELD_NAME))) - .patch(List.of(patchOp)) - .build() - .getJsonPatch()) - .auditStamp(mclItem.getAuditStamp()) - .systemMetadata(mclItem.getSystemMetadata()) - .build(retrieverContext.getAspectRetriever().getEntityRegistry())); + patchOpMap + .computeIfAbsent(entity.getSourceUrn(), urn -> new ArrayList<>()) + .add(patchOp); } } } } + for (String urn : patchOpMap.keySet()) { + EntitySpec entitySpec = + retrieverContext + .getAspectRetriever() + .getEntityRegistry() + .getEntitySpec(DATA_PRODUCT_ENTITY_NAME); + mcpItems.add( + PatchItemImpl.builder() + .urn(UrnUtils.getUrn(urn)) + .entitySpec( + retrieverContext + .getAspectRetriever() + .getEntityRegistry() + .getEntitySpec(DATA_PRODUCT_ENTITY_NAME)) + .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME) + .aspectSpec(entitySpec.getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys( + Map.of( + DataProductPropertiesTemplate.ASSETS_FIELD_NAME, + List.of(DataProductPropertiesTemplate.KEY_FIELD_NAME))) + .patch(patchOpMap.get(urn)) + .build() + .getJsonPatch()) + .auditStamp(mclItem.getAuditStamp()) + .systemMetadata(mclItem.getSystemMetadata()) + .build(retrieverContext.getAspectRetriever().getEntityRegistry())); + } return mcpItems.stream(); } return Stream.empty(); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java b/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java index 1151014bf1162f..12dd57f94da23d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java @@ -34,6 +34,8 @@ import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.test.metadata.aspect.TestEntityRegistry; import io.datahubproject.metadata.context.RetrieverContext; +import jakarta.json.JsonArray; +import jakarta.json.JsonObject; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -251,6 +253,111 @@ public void testDPRemoveOld() { .build(mockAspectRetriever.getEntityRegistry()))); } + @Test + public void testBulkAssetMove() { + DataProductUnsetSideEffect test = new DataProductUnsetSideEffect(); + test.setConfig(TEST_PLUGIN_CONFIG); + + // Create 100 dataset URNs and set up their existing relationships + List datasetUrns = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + Urn datasetUrn = + UrnUtils.getUrn( + String.format("urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_%d,PROD)", i)); + datasetUrns.add(datasetUrn); + + // Mock the existing relationship for each dataset with the old data product + RelatedEntities relatedEntities = + new RelatedEntities( + "DataProductContains", + TEST_PRODUCT_URN_2.toString(), // Old data product + datasetUrn.toString(), + RelationshipDirection.INCOMING, + null); + + List relatedEntitiesList = new ArrayList<>(); + relatedEntitiesList.add(relatedEntities); + RelatedEntitiesScrollResult relatedEntitiesScrollResult = + new RelatedEntitiesScrollResult(1, 10, null, relatedEntitiesList); + + when(retrieverContext + .getGraphRetriever() + .scrollRelatedEntities( + eq(null), + eq(QueryUtils.newFilter("urn", datasetUrn.toString())), + eq(null), + eq(EMPTY_FILTER), + eq(ImmutableList.of("DataProductContains")), + eq( + QueryUtils.newRelationshipFilter( + EMPTY_FILTER, RelationshipDirection.INCOMING)), + eq(Collections.emptyList()), + eq(null), + eq(10), + eq(null), + eq(null))) + .thenReturn(relatedEntitiesScrollResult); + } + + // Create data product properties with all 100 assets + DataProductProperties dataProductProperties = new DataProductProperties(); + DataProductAssociationArray dataProductAssociations = new DataProductAssociationArray(); + for (Urn datasetUrn : datasetUrns) { + DataProductAssociation association = new DataProductAssociation(); + association.setDestinationUrn(datasetUrn); + dataProductAssociations.add(association); + } + dataProductProperties.setAssets(dataProductAssociations); + + // Run test + ChangeItemImpl dataProductPropertiesChangeItem = + ChangeItemImpl.builder() + .urn(TEST_PRODUCT_URN) // New data product + .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME) + .changeType(ChangeType.UPSERT) + .entitySpec(TEST_REGISTRY.getEntitySpec(DATA_PRODUCT_ENTITY_NAME)) + .aspectSpec( + TEST_REGISTRY + .getEntitySpec(DATA_PRODUCT_ENTITY_NAME) + .getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)) + .recordTemplate(dataProductProperties) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(mockAspectRetriever); + + List testOutput = + test.postMCPSideEffect( + List.of( + MCLItemImpl.builder() + .build( + dataProductPropertiesChangeItem, + null, + null, + retrieverContext.getAspectRetriever())), + retrieverContext) + .toList(); + + // Verify test + assertEquals(testOutput.size(), 1, "Expected one patch to remove assets from old data product"); + + MCPItem patchItem = testOutput.get(0); + assertEquals( + patchItem.getUrn(), TEST_PRODUCT_URN_2, "Patch should target the old data product"); + assertEquals(patchItem.getAspectName(), DATA_PRODUCT_PROPERTIES_ASPECT_NAME); + + // Verify the patch contains remove operations for all 100 assets + JsonArray patchArray = ((PatchItemImpl) patchItem).getPatch().toJsonArray(); + assertEquals(patchArray.size(), 100, "Should have 100 remove operations"); + + // Verify each remove operation + for (int i = 0; i < 100; i++) { + JsonObject op = patchArray.getJsonObject(i); + assertEquals(op.getString("op"), PatchOperationType.REMOVE.getValue()); + assertEquals( + op.getString("path"), + String.format("/assets/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_%d,PROD)", i)); + } + } + private static DataProductProperties getTestDataProductProperties(Urn destinationUrn) { DataProductProperties dataProductProperties = new DataProductProperties(); DataProductAssociationArray dataProductAssociations = new DataProductAssociationArray(); From 9f9a8b1006afbf92c01f558db28503728005dc29 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 26 Nov 2024 01:00:56 -0500 Subject: [PATCH 068/174] fix(ingest/tableau): handle none database field (#11950) --- .../src/datahub/ingestion/source/tableau/tableau_common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py index 8d6746b6433a4e..ac917c5f128ed2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py @@ -643,8 +643,11 @@ def create( cls, d: dict, default_schema_map: Optional[Dict[str, str]] = None ) -> "TableauUpstreamReference": # Values directly from `table` object from Tableau - database = t_database = d.get(c.DATABASE, {}).get(c.NAME) - database_id = d.get(c.DATABASE, {}).get(c.ID) + database_dict = ( + d.get(c.DATABASE) or {} + ) # this sometimes is None, so we need the `or {}` + database = t_database = database_dict.get(c.NAME) + database_id = database_dict.get(c.ID) schema = t_schema = d.get(c.SCHEMA) table = t_table = d.get(c.NAME) or "" t_full_name = d.get(c.FULL_NAME) From 301e5cd18f50b7e4e91962e6be4957870809d3ac Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 26 Nov 2024 00:02:06 -0600 Subject: [PATCH 069/174] fix(urn-validator): update urn validation logic (#11952) --- .../entity/validation/ValidationApiUtils.java | 52 ++++++++++++++++++- .../validation/ValidationApiUtilsTest.java | 19 ++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java index adc539cd926d32..c2e1c47eca1fdb 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java @@ -115,7 +115,8 @@ public static void validateUrn( /** Recursively process URN parts with URL decoding */ private static Stream processUrnPartRecursively(String urnPart) { - String decodedPart = URLDecoder.decode(urnPart, StandardCharsets.UTF_8); + String decodedPart = + URLDecoder.decode(URLEncodingFixer.fixURLEncoding(urnPart), StandardCharsets.UTF_8); if (decodedPart.startsWith("urn:li:")) { // Recursively process nested URN after decoding return UrnUtils.getUrn(decodedPart).getEntityKey().getParts().stream() @@ -177,4 +178,53 @@ public static void validateRecordTemplate( RecordTemplateValidator.validate(aspect, resultFunction, validator); } } + + /** + * Fixes malformed URL encoding by escaping unescaped % characters while preserving valid + * percent-encoded sequences. + */ + private static class URLEncodingFixer { + /** + * @param input The potentially malformed URL-encoded string + * @return A string with proper URL encoding that can be safely decoded + */ + public static String fixURLEncoding(String input) { + if (input == null) { + return null; + } + + StringBuilder result = new StringBuilder(input.length() * 2); + int i = 0; + + while (i < input.length()) { + char currentChar = input.charAt(i); + + if (currentChar == '%') { + if (i + 2 < input.length()) { + // Check if the next two characters form a valid hex pair + String hexPair = input.substring(i + 1, i + 3); + if (isValidHexPair(hexPair)) { + // This is a valid percent-encoded sequence, keep it as is + result.append(currentChar); + } else { + // Invalid sequence, escape the % character + result.append("%25"); + } + } else { + // % at the end of string, escape it + result.append("%25"); + } + } else { + result.append(currentChar); + } + i++; + } + + return result.toString(); + } + + private static boolean isValidHexPair(String pair) { + return pair.matches("[0-9A-Fa-f]{2}"); + } + } } diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java index 7a671a0a496309..e683e594d8766c 100644 --- a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java +++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java @@ -89,7 +89,24 @@ public void testValidComplexUrn() { @Test(expectedExceptions = NullPointerException.class) public void testUrnNull() { - // Act ValidationApiUtils.validateUrn(entityRegistry, null); } + + @Test + public void testValidPartialUrlEncode() { + Urn validUrn = UrnUtils.getUrn("urn:li:assertion:123=-%28__% weekly__%29"); + + ValidationApiUtils.validateUrn(entityRegistry, validUrn); + // If no exception is thrown, test passes + } + + @Test + public void testValidPartialUrlEncode2() { + Urn validUrn = + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:s3,urn:li:dataset:%28urn:li:dataPlatform:s3%2Ctest-datalake-concepts%prog_maintenance%2CPROD%29,PROD)"); + + ValidationApiUtils.validateUrn(entityRegistry, validUrn); + // If no exception is thrown, test passes + } } From 3423e348992e614c48043f8656c8d066f0dae66e Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:22:06 +0530 Subject: [PATCH 070/174] feat(ingest): add more logs for kafka polling (#11954) --- metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py index 06d929774240ba..e57dc853a83c65 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py @@ -157,7 +157,9 @@ def get_kafka_consumer( if CallableConsumerConfig.is_callable_config(connection.consumer_config): # As per documentation, we need to explicitly call the poll method to make sure OAuth callback gets executed # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration + logger.debug("Initiating polling for kafka consumer") consumer.poll(timeout=30) + logger.debug("Initiated polling for kafka consumer") return consumer From b357f87f94e766da3d5962d893b759a002357d44 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Tue, 26 Nov 2024 18:56:30 +0530 Subject: [PATCH 071/174] fix(ingest/sigma): migrate sigma workbooks from container to dashboard (#11939) --- .../docs/sources/sigma/sigma_pre.md | 2 +- .../ingestion/source/sigma/data_classes.py | 1 + .../datahub/ingestion/source/sigma/sigma.py | 144 +- .../golden_test_platform_instance_ingest.json | 1411 ++++++++--------- .../sigma/golden_test_sigma_ingest.json | 419 ++--- ...est_sigma_ingest_shared_entities_mces.json | 323 ++-- 6 files changed, 1055 insertions(+), 1245 deletions(-) diff --git a/metadata-ingestion/docs/sources/sigma/sigma_pre.md b/metadata-ingestion/docs/sources/sigma/sigma_pre.md index 382a2fe67b944d..433f85a69f907c 100644 --- a/metadata-ingestion/docs/sources/sigma/sigma_pre.md +++ b/metadata-ingestion/docs/sources/sigma/sigma_pre.md @@ -16,7 +16,7 @@ This source extracts the following: | Sigma | Datahub | Notes | |------------------------|---------------------------------------------------------------|----------------------------------| | `Workspace` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workspace"` | -| `Workbook` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workbook"` | +| `Workbook` | [Dashboard](../../metamodel/entities/dashboard.md) | SubType `"Sigma Workbook"` | | `Page` | [Dashboard](../../metamodel/entities/dashboard.md) | | | `Element` | [Chart](../../metamodel/entities/chart.md) | | | `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Sigma Dataset"` | diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py index 922b0be3b4a93c..5a657d804cb7bf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py @@ -80,6 +80,7 @@ class Workbook(BaseModel): path: str latestVersion: int workspaceId: Optional[str] = None + description: Optional[str] = None pages: List[Page] = [] badge: Optional[str] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index dd4b65a2cbdf29..e96eeb58d96efe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -4,7 +4,12 @@ import datahub.emitter.mce_builder as builder from datahub.configuration.common import ConfigurationError from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers +from datahub.emitter.mcp_builder import ( + add_entity_to_container, + add_owner_to_entity_wu, + add_tags_to_entity_wu, + gen_containers, +) from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -59,12 +64,14 @@ UpstreamLineage, ) from datahub.metadata.schema_classes import ( + AuditStampClass, BrowsePathEntryClass, BrowsePathsV2Class, ChangeAuditStampsClass, ChartInfoClass, DashboardInfoClass, DataPlatformInstanceClass, + EdgeClass, GlobalTagsClass, InputFieldClass, InputFieldsClass, @@ -74,6 +81,7 @@ SchemaFieldClass, SchemaFieldDataTypeClass, StringTypeClass, + SubTypesClass, TagAssociationClass, ) from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result @@ -257,11 +265,6 @@ def _gen_entity_browsepath_aspect( entries = [ BrowsePathEntryClass(id=parent_entity_urn, urn=parent_entity_urn) ] + [BrowsePathEntryClass(id=path) for path in paths] - if self.config.platform_instance: - urn = builder.make_dataplatform_instance_urn( - self.platform, self.config.platform_instance - ) - entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries return MetadataChangeProposalWrapper( entityUrn=entity_urn, aspect=BrowsePathsV2Class(entries), @@ -424,11 +427,11 @@ def _gen_elements_workunit( elements: List[Element], workbook: Workbook, all_input_fields: List[InputFieldClass], + paths: List[str], ) -> Iterable[MetadataWorkUnit]: """ Map Sigma page element to Datahub Chart """ - for element in elements: chart_urn = builder.make_chart_urn( platform=self.platform, @@ -459,11 +462,14 @@ def _gen_elements_workunit( ), ).as_workunit() - yield from add_entity_to_container( - container_key=self._gen_workbook_key(workbook.workbookId), - entity_type="chart", - entity_urn=chart_urn, - ) + if workbook.workspaceId: + yield self._gen_entity_browsepath_aspect( + entity_urn=chart_urn, + parent_entity_urn=builder.make_container_urn( + self._gen_workspace_key(workbook.workspaceId) + ), + paths=paths + [workbook.name], + ) # Add sigma dataset's upstream dataset urn mapping for dataset_urn, upstream_dataset_urns in inputs.items(): @@ -494,7 +500,9 @@ def _gen_elements_workunit( all_input_fields.extend(element_input_fields) - def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: + def _gen_pages_workunit( + self, workbook: Workbook, paths: List[str] + ) -> Iterable[MetadataWorkUnit]: """ Map Sigma workbook page to Datahub dashboard """ @@ -505,20 +513,23 @@ def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: yield self._gen_dashboard_info_workunit(page) - yield from add_entity_to_container( - container_key=self._gen_workbook_key(workbook.workbookId), - entity_type="dashboard", - entity_urn=dashboard_urn, - ) - dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn) if dpi_aspect: yield dpi_aspect all_input_fields: List[InputFieldClass] = [] + if workbook.workspaceId: + yield self._gen_entity_browsepath_aspect( + entity_urn=dashboard_urn, + parent_entity_urn=builder.make_container_urn( + self._gen_workspace_key(workbook.workspaceId) + ), + paths=paths + [workbook.name], + ) + yield from self._gen_elements_workunit( - page.elements, workbook, all_input_fields + page.elements, workbook, all_input_fields, paths ) yield MetadataChangeProposalWrapper( @@ -531,42 +542,89 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni Map Sigma Workbook to Datahub container """ owner_username = self.sigma_api.get_user_name(workbook.createdBy) - workbook_key = self._gen_workbook_key(workbook.workbookId) - yield from gen_containers( - container_key=workbook_key, - name=workbook.name, - sub_types=[BIContainerSubTypes.SIGMA_WORKBOOK], - parent_container_key=( - self._gen_workspace_key(workbook.workspaceId) - if workbook.workspaceId - else None + + dashboard_urn = self._gen_dashboard_urn(workbook.workbookId) + + yield self._gen_entity_status_aspect(dashboard_urn) + + lastModified = AuditStampClass( + time=int(workbook.updatedAt.timestamp() * 1000), + actor="urn:li:corpuser:datahub", + ) + created = AuditStampClass( + time=int(workbook.createdAt.timestamp() * 1000), + actor="urn:li:corpuser:datahub", + ) + + dashboard_info_cls = DashboardInfoClass( + title=workbook.name, + description=workbook.description if workbook.description else "", + dashboards=[ + EdgeClass( + destinationUrn=self._gen_dashboard_urn(page.get_urn_part()), + sourceUrn=dashboard_urn, + ) + for page in workbook.pages + ], + externalUrl=workbook.url, + lastModified=ChangeAuditStampsClass( + created=created, lastModified=lastModified ), - extra_properties={ + customProperties={ "path": workbook.path, "latestVersion": str(workbook.latestVersion), }, - owner_urn=( - builder.make_user_urn(owner_username) - if self.config.ingest_owner and owner_username - else None - ), - external_url=workbook.url, - tags=[workbook.badge] if workbook.badge else None, - created=int(workbook.createdAt.timestamp() * 1000), - last_modified=int(workbook.updatedAt.timestamp() * 1000), ) + yield MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, aspect=dashboard_info_cls + ).as_workunit() + + # Set subtype + yield MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, + aspect=SubTypesClass(typeNames=[BIContainerSubTypes.SIGMA_WORKBOOK]), + ).as_workunit() + + # Ownership + owner_urn = ( + builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None + ) + if owner_urn: + yield from add_owner_to_entity_wu( + entity_type="dashboard", + entity_urn=dashboard_urn, + owner_urn=owner_urn, + ) + + # Tags + tags = [workbook.badge] if workbook.badge else None + if tags: + yield from add_tags_to_entity_wu( + entity_type="dashboard", + entity_urn=dashboard_urn, + tags=sorted(tags), + ) paths = workbook.path.split("/")[1:] - if len(paths) > 0 and workbook.workspaceId: + if workbook.workspaceId: yield self._gen_entity_browsepath_aspect( - entity_urn=builder.make_container_urn(workbook_key), + entity_urn=dashboard_urn, parent_entity_urn=builder.make_container_urn( self._gen_workspace_key(workbook.workspaceId) ), - paths=paths, + paths=paths + [workbook.name], ) - yield from self._gen_pages_workunit(workbook) + if len(paths) == 0: + yield from add_entity_to_container( + container_key=self._gen_workspace_key(workbook.workspaceId), + entity_type="dashboard", + entity_urn=dashboard_urn, + ) + + yield from self._gen_pages_workunit(workbook, paths) def _gen_sigma_dataset_upstream_lineage_workunit( self, diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json index 12bb7734f30a63..645e710309b0da 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json @@ -10,23 +10,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496006, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - } - }, - "systemMetadata": { - "lastObserved": 1713794496008, + "lastObserved": 1732608523763, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -56,7 +40,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496007, + "lastObserved": 1732608523764, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -65,40 +49,30 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" } }, "systemMetadata": { - "lastObserved": 1713794496008, + "lastObserved": 1732608523764, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - } - ] + "removed": false } }, "systemMetadata": { - "lastObserved": 1713794496010, + "lastObserved": 1732608523833, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -107,16 +81,15 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "Sigma Dataset" - ] + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" } }, "systemMetadata": { - "lastObserved": 1713794496009, + "lastObserved": 1732608523764, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -125,40 +98,44 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "subTypes", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "typeNames": [ + "Sigma Dataset" + ] } }, "systemMetadata": { - "lastObserved": 1713794496009, + "lastObserved": 1732608523765, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" + } + ] } }, "systemMetadata": { - "lastObserved": 1713794496011, + "lastObserved": 1732608523835, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -167,14 +144,14 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "status", "aspect": { "json": { - "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + "removed": false } }, "systemMetadata": { - "lastObserved": 1713794496012, + "lastObserved": 1732608523781, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -206,33 +183,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496012, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496201, + "lastObserved": 1732608523781, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -241,49 +192,26 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - } - }, - "systemMetadata": { - "lastObserved": 1713794496013, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "container", "aspect": { "json": { - "typeNames": [ - "Sigma Workspace" - ] + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" } }, "systemMetadata": { - "lastObserved": 1713794496200, + "lastObserved": 1732608523782, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" @@ -291,22 +219,19 @@ { "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "New Folder" } ] } }, "systemMetadata": { - "lastObserved": 1713794496015, + "lastObserved": 1732608523765, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -316,7 +241,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496200, + "lastObserved": 1732608523782, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -334,34 +259,51 @@ } }, "systemMetadata": { - "lastObserved": 1713794496014, + "lastObserved": 1732608523783, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "chartInfo", "aspect": { "json": { - "tags": [ + "customProperties": { + "VizualizationType": "bar", + "type": "visualization" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", + "title": "Count of Profile Id by Status", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "tag": "urn:li:tag:Deprecated" + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" } ] } }, "systemMetadata": { - "lastObserved": 1713794496015, + "lastObserved": 1732608523833, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -380,14 +322,14 @@ } }, "systemMetadata": { - "lastObserved": 1713794496014, + "lastObserved": 1732608523765, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -396,57 +338,76 @@ } }, "systemMetadata": { - "lastObserved": 1713794496199, + "lastObserved": 1732608523784, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "dashboardInfo", "aspect": { "json": { "customProperties": { - "platform": "sigma", - "instance": "cloud_instance", - "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" - }, - "name": "Acryl Data", - "created": { - "time": 1710232264826 + "path": "Acryl Data", + "latestVersion": "2" }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "title": "Acryl Workbook", + "description": "", + "charts": [], + "datasets": [], + "dashboards": [ + { + "sourceUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)" + }, + { + "sourceUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)" + } + ], "lastModified": { - "time": 1710232264826 + "created": { + "time": 1713188691477, + "actor": "urn:li:corpuser:datahub" + }, + "lastModified": { + "time": 1713189117302, + "actor": "urn:li:corpuser:datahub" + } } } }, "systemMetadata": { - "lastObserved": 1713794496199, + "lastObserved": 1732608523785, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Sigma Workbook" + ] } }, "systemMetadata": { - "lastObserved": 1713794496053, + "lastObserved": 1732608523785, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -455,227 +416,81 @@ { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "New Folder" } ] } }, "systemMetadata": { - "lastObserved": 1713794496202, + "lastObserved": 1732608523783, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "ownership", "aspect": { "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { - "lastObserved": 1713794496055, + "lastObserved": 1732608523786, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "inputFields", + "aspectName": "globalTags", "aspect": { "json": { - "fields": [ + "tags": [ { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)", - "schemaField": { - "fieldPath": "Pk", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } - }, - { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)", - "schemaField": { - "fieldPath": "Pet Fk", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } - }, - { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)", - "schemaField": { - "fieldPath": "Human Fk", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } - }, - { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)", - "schemaField": { - "fieldPath": "Status", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } - }, - { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)", - "schemaField": { - "fieldPath": "Created At", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } - }, - { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)", - "schemaField": { - "fieldPath": "Updated At", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496055, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "json": { - "customProperties": { - "VizualizationType": "levelTable", - "type": "table" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", - "title": "ADOPTIONS", - "description": "", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718348049212, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "tag": "urn:li:tag:Warning" } ] } }, "systemMetadata": { - "lastObserved": 1713794496058, + "lastObserved": 1732608523786, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "container", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" } }, "systemMetadata": { - "lastObserved": 1713794496019, + "lastObserved": 1732608523786, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -691,7 +506,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496021, + "lastObserved": 1732608523787, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -713,6 +528,7 @@ "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -726,87 +542,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496022, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "sigma", - "instance": "cloud_instance", - "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", - "path": "Acryl Data", - "latestVersion": "2" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", - "name": "Acryl Workbook", - "created": { - "time": 1713188691477 - }, - "lastModified": { - "time": 1713189117302 - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496016, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - }, - "systemMetadata": { - "lastObserved": 1713794496023, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Sigma Workbook" - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496018, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496017, + "lastObserved": 1732608523788, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -823,7 +559,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496023, + "lastObserved": 1732608523788, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -845,108 +581,87 @@ "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" }, { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1713794496024, + "lastObserved": 1732608523788, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "browsePathsV2", "aspect": { "json": { - "tags": [ + "path": [ { - "tag": "urn:li:tag:Warning" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1713794496019, + "lastObserved": 1732608523787, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - } - }, - "systemMetadata": { - "lastObserved": 1713794496018, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496114, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "globalTags", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, + "tags": [ { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + "tag": "urn:li:tag:Deprecated" } ] } }, "systemMetadata": { - "lastObserved": 1713794496020, + "lastObserved": 1732608523783, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "ownership", "aspect": { "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { - "lastObserved": 1713794496116, + "lastObserved": 1732608523782, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1047,208 +762,20 @@ "nativeDataType": "String", "recursive": false, "isPartOfKey": false - } - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496117, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "json": { - "customProperties": { - "VizualizationType": "bar", - "type": "visualization" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", - "title": "Count of Profile Id by Status", - "description": "", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718348049268, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496119, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496124, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "json": { - "customProperties": { - "ElementsCount": "1" - }, - "title": "Page 2", - "description": "", - "charts": [ - "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496125, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - } - }, - "systemMetadata": { - "lastObserved": 1713794496020, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - }, - "systemMetadata": { - "lastObserved": 1713794496125, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "json": { - "customProperties": { - "VizualizationType": "levelTable", - "type": "table" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", - "title": "PETS ADOPTIONS JOIN", - "description": "", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" + } } ] } }, "systemMetadata": { - "lastObserved": 1718348049351, + "lastObserved": 1732608523834, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -1257,69 +784,179 @@ } }, "systemMetadata": { - "lastObserved": 1713794496188, + "lastObserved": 1732608523803, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "browsePathsV2", "aspect": { "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" + } + ] } }, "systemMetadata": { - "lastObserved": 1713794496126, + "lastObserved": 1732608523806, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "chartInfo", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", + "title": "ADOPTIONS", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" } ] } }, "systemMetadata": { - "lastObserved": 1713794496126, + "lastObserved": 1732608523804, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "inputFields", "aspect": { "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)", + "schemaField": { + "fieldPath": "Pk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)", + "schemaField": { + "fieldPath": "Pet Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)", + "schemaField": { + "fieldPath": "Human Fk", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)", + "schemaField": { + "fieldPath": "Status", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)", + "schemaField": { + "fieldPath": "Created At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] } }, "systemMetadata": { - "lastObserved": 1713794496189, + "lastObserved": 1732608523804, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1483,40 +1120,219 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", - "schemaField": { - "fieldPath": "Updated At", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732608523836, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1732608523838, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "1" + }, + "title": "Page 2", + "description": "", + "charts": [ + "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523838, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1732608523839, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", - "schemaField": { - "fieldPath": "Count of Profile Id", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732608523839, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)", + "type": "COPY" } ] } }, "systemMetadata": { - "lastObserved": 1713794496120, + "lastObserved": 1732608523874, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Deprecated", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Deprecated" + } + }, + "systemMetadata": { + "lastObserved": 1732608523874, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1732608523872, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523873, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1698,7 +1514,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496194, + "lastObserved": 1732608523870, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1707,27 +1523,54 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1732608523866, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", + "title": "PETS ADOPTIONS JOIN", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" }, { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" } ] } }, "systemMetadata": { - "lastObserved": 1713794496194, + "lastObserved": 1732608523866, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1909,64 +1752,134 @@ } }, "systemMetadata": { - "lastObserved": 1713794496190, + "lastObserved": 1732608523867, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "upstreams": [ + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1732608523873, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)", - "type": "COPY" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" } ] } }, "systemMetadata": { - "lastObserved": 1718348049380, + "lastObserved": 1732608523874, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:Warning", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", "changeType": "UPSERT", - "aspectName": "tagKey", + "aspectName": "browsePathsV2", "aspect": { "json": { - "name": "Warning" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732608523869, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "instance": "cloud_instance", + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" + }, + "name": "Acryl Data", + "created": { + "time": 1710232264826 + }, + "lastModified": { + "time": 1710232264826 + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523872, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workspace" + ] } }, "systemMetadata": { - "lastObserved": 1713794496203, + "lastObserved": 1732608523873, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "tag", - "entityUrn": "urn:li:tag:Deprecated", + "entityUrn": "urn:li:tag:Warning", "changeType": "UPSERT", "aspectName": "tagKey", "aspect": { "json": { - "name": "Deprecated" + "name": "Warning" } }, "systemMetadata": { - "lastObserved": 1713794496203, + "lastObserved": 1732608523875, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json index f800cb19f88115..bb37e7029330b2 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json @@ -261,37 +261,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "sigma", - "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", - "path": "Acryl Data", - "latestVersion": "2" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", - "name": "Acryl Workbook", - "created": { - "time": 1713188691477 - }, - "lastModified": { - "time": 1713189117302 - } - } - }, - "systemMetadata": { - "lastObserved": 1713795619227, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -300,7 +271,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619228, + "lastObserved": 1732513099680, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -338,32 +309,70 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ + "typeNames": [ + "Sigma Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1732513099681, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "path": "Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "title": "Acryl Workbook", + "description": "", + "charts": [], + "datasets": [], + "dashboards": [ { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)" }, { - "id": "New Folder" + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)" } - ] + ], + "lastModified": { + "created": { + "time": 1713188691477, + "actor": "urn:li:corpuser:datahub" + }, + "lastModified": { + "time": 1713189117302, + "actor": "urn:li:corpuser:datahub" + } + } } }, "systemMetadata": { - "lastObserved": 1713795619226, + "lastObserved": 1732535135915, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -382,41 +391,51 @@ } }, "systemMetadata": { - "lastObserved": 1713795619229, + "lastObserved": 1732513099681, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "globalTags", "aspect": { "json": { - "typeNames": [ - "Sigma Workbook" + "tags": [ + { + "tag": "urn:li:tag:Warning" + } ] } }, "systemMetadata": { - "lastObserved": 1713795619229, + "lastObserved": 1732513099682, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "browsePathsV2", "aspect": { "json": { - "platform": "urn:li:dataPlatform:sigma" + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "New Folder" + } + ] } }, "systemMetadata": { - "lastObserved": 1713795619228, + "lastObserved": 1713795619226, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -457,79 +476,62 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", "changeType": "UPSERT", - "aspectName": "dashboardInfo", + "aspectName": "browsePathsV2", "aspect": { "json": { - "customProperties": { - "ElementsCount": "2" - }, - "title": "Page 1", - "description": "", - "charts": [ - "urn:li:chart:(sigma,kH0MeihtGs)", - "urn:li:chart:(sigma,Ml9C5ezT5W)" - ], - "datasets": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + { + "id": "Acryl Workbook" } - } + ] } }, "systemMetadata": { - "lastObserved": 1713795619233, + "lastObserved": 1732545848809, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "container", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" } }, "systemMetadata": { - "lastObserved": 1713795619224, + "lastObserved": 1732513099682, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "browsePathsV2", "aspect": { "json": { - "tags": [ + "path": [ { - "tag": "urn:li:tag:Warning" + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1713795619230, + "lastObserved": 1732545848807, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -538,14 +540,34 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dashboardInfo", "aspect": { "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "customProperties": { + "ElementsCount": "2" + }, + "title": "Page 1", + "description": "", + "charts": [ + "urn:li:chart:(sigma,kH0MeihtGs)", + "urn:li:chart:(sigma,Ml9C5ezT5W)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } }, "systemMetadata": { - "lastObserved": 1713795619234, + "lastObserved": 1713795619233, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -554,59 +576,44 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "ownership", "aspect": { "json": { - "tags": [ + "owners": [ { - "tag": "urn:li:tag:Deprecated" + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" } - ] + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { - "lastObserved": 1713795619226, + "lastObserved": 1713795619224, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "globalTags", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - }, + "tags": [ { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "tag": "urn:li:tag:Deprecated" } ] } }, "systemMetadata": { - "lastObserved": 1713795619234, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - } - }, - "systemMetadata": { - "lastObserved": 1713795619231, + "lastObserved": 1713795619226, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -642,51 +649,13 @@ "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" }, { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713795619375, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1713795619231, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619373, + "lastObserved": 1732545848872, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -883,6 +852,7 @@ "urn:li:chart:(sigma,tQJu5N1l81)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -901,6 +871,30 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545848877, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -914,14 +908,13 @@ "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" }, { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1713795619270, + "lastObserved": 1732545848829, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1179,54 +1172,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619382, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619267, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619449, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", @@ -1409,31 +1354,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - }, - { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713795619383, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -1554,14 +1474,13 @@ "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" }, { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1713795619453, + "lastObserved": 1732545848921, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json index d6b702bdfd6695..1ce671f09d7765 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json @@ -279,37 +279,24 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "sigma", - "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", - "path": "New Acryl Data", - "latestVersion": "2" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", - "name": "Acryl Workbook", - "created": { - "time": 1713188691477 - }, - "lastModified": { - "time": 1713189117302 - } + "removed": false } }, "systemMetadata": { - "lastObserved": 1718004101680, + "lastObserved": 1732513100094, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -318,48 +305,76 @@ } }, "systemMetadata": { - "lastObserved": 1718004101680, + "lastObserved": 1718004101684, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "subTypes", "aspect": { "json": { - "platform": "urn:li:dataPlatform:sigma" + "typeNames": [ + "Sigma Workbook" + ] } }, "systemMetadata": { - "lastObserved": 1718004101681, + "lastObserved": 1732513100095, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dashboardInfo", "aspect": { "json": { - "typeNames": [ - "Sigma Workbook" - ] + "customProperties": { + "path": "New Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "title": "Acryl Workbook", + "description": "", + "charts": [], + "datasets": [], + "dashboards": [ + { + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)" + }, + { + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)" + } + ], + "lastModified": { + "created": { + "time": 1713188691477, + "actor": "urn:li:corpuser:datahub" + }, + "lastModified": { + "time": 1713189117302, + "actor": "urn:li:corpuser:datahub" + } + } } }, "systemMetadata": { - "lastObserved": 1718004101681, + "lastObserved": 1732535136409, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -378,14 +393,14 @@ } }, "systemMetadata": { - "lastObserved": 1718004101682, + "lastObserved": 1732513100096, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "globalTags", "aspect": { @@ -398,60 +413,7 @@ } }, "systemMetadata": { - "lastObserved": 1718004101683, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b83da80a4d444484521d9f7aca958742" - } - }, - "systemMetadata": { - "lastObserved": 1718004101683, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718004101684, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1718004101684, + "lastObserved": 1732513100096, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -473,6 +435,7 @@ "urn:li:chart:(sigma,Ml9C5ezT5W)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -491,47 +454,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1718004101686, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - }, - { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718004101686, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -582,17 +504,65 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545849249, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "container": "urn:li:container:b83da80a4d444484521d9f7aca958742" } }, "systemMetadata": { - "lastObserved": 1718004101689, + "lastObserved": 1732513100096, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545849248, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -717,14 +687,13 @@ "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" }, { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1718004101692, + "lastObserved": 1732545849252, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -778,22 +747,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1718004101695, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", @@ -914,14 +867,13 @@ "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" }, { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1718004101697, + "lastObserved": 1732545849255, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1155,6 +1107,7 @@ "urn:li:chart:(sigma,tQJu5N1l81)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -1174,17 +1127,17 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "status", "aspect": { "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "removed": false } }, "systemMetadata": { - "lastObserved": 1718004101704, + "lastObserved": 1718004101706, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1202,30 +1155,13 @@ "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" }, { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1718004101704, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1718004101706, + "lastObserved": 1732545849260, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1263,22 +1199,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1718004101708, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", @@ -1474,14 +1394,13 @@ "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" }, { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1718004101712, + "lastObserved": 1732545849264, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } From 01fb64c531f6801909d1f4704c44afb43f5ea05f Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Tue, 26 Nov 2024 15:32:03 +0100 Subject: [PATCH 072/174] fix(ingest/bigquery): Fix performance issue with column profiling ignore (#11807) --- .../source/bigquery_v2/bigquery_schema.py | 1 - .../source/bigquery_v2/bigquery_schema_gen.py | 21 ------------- .../ingestion/source/bigquery_v2/profiler.py | 6 ---- .../ingestion/source/ge_data_profiler.py | 31 ++++++++++++++----- .../ingestion/source/ge_profiling_config.py | 5 +++ 5 files changed, 28 insertions(+), 36 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index be85d037af9cd1..3ce34be8dc89df 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -118,7 +118,6 @@ class BigqueryTable(BaseTable): active_billable_bytes: Optional[int] = None long_term_billable_bytes: Optional[int] = None partition_info: Optional[PartitionInfo] = None - columns_ignore_from_profiling: List[str] = field(default_factory=list) external: bool = False constraints: List[BigqueryTableConstraint] = field(default_factory=list) table_type: Optional[str] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py index 788016103d0fcd..4a3b47f6b543a6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema_gen.py @@ -598,18 +598,6 @@ def _process_schema( dataset_name=dataset_name, ) - # This method is used to generate the ignore list for datatypes the profiler doesn't support we have to do it here - # because the profiler doesn't have access to columns - def generate_profile_ignore_list(self, columns: List[BigqueryColumn]) -> List[str]: - ignore_list: List[str] = [] - for column in columns: - if not column.data_type or any( - word in column.data_type.lower() - for word in ["array", "struct", "geography", "json"] - ): - ignore_list.append(column.field_path) - return ignore_list - def _process_table( self, table: BigqueryTable, @@ -631,15 +619,6 @@ def _process_table( ) table.column_count = len(columns) - # We only collect profile ignore list if profiling is enabled and profile_table_level_only is false - if ( - self.config.is_profiling_enabled() - and not self.config.profiling.profile_table_level_only - ): - table.columns_ignore_from_profiling = self.generate_profile_ignore_list( - columns - ) - if not table.column_count: logger.warning( f"Table doesn't have any column or unable to get columns for table: {table_identifier}" diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py index 6af8166fbf70c3..182ae2265cb162 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py @@ -166,12 +166,6 @@ def get_workunits( normalized_table_name = BigqueryTableIdentifier( project_id=project_id, dataset=dataset, table=table.name ).get_table_name() - for column in table.columns_ignore_from_profiling: - # Profiler has issues with complex types (array, struct, geography, json), so we deny those types from profiling - # We also filter columns without data type as it means that column is part of a complex type. - self.config.profile_pattern.deny.append( - f"^{normalized_table_name}.{column}$" - ) if table.external and not self.config.profiling.profile_external_tables: self.report.profiling_skipped_other[f"{project_id}.{dataset}"] += 1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index 0d9dadaf6af555..f7d783cd3dec0b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -7,6 +7,7 @@ import functools import json import logging +import re import threading import traceback import unittest.mock @@ -123,6 +124,8 @@ _datasource_connection_injection_lock = threading.Lock() +NORMALIZE_TYPE_PATTERN = re.compile(r"^(.*?)(?:[\[<(].*)?$") + @contextlib.contextmanager def _inject_connection_into_datasource(conn: Connection) -> Iterator[None]: @@ -165,11 +168,9 @@ def get_column_unique_count_dh_patch(self: SqlAlchemyDataset, column: str) -> in return convert_to_json_serializable(element_values.fetchone()[0]) elif self.engine.dialect.name.lower() == BIGQUERY: element_values = self.engine.execute( - sa.select( - [ - sa.func.coalesce(sa.text(f"APPROX_COUNT_DISTINCT(`{column}`)")), - ] - ).select_from(self._table) + sa.select(sa.func.APPROX_COUNT_DISTINCT(sa.column(column))).select_from( + self._table + ) ) return convert_to_json_serializable(element_values.fetchone()[0]) elif self.engine.dialect.name.lower() == SNOWFLAKE: @@ -378,6 +379,9 @@ def _get_columns_to_profile(self) -> List[str]: f"{self.dataset_name}.{col}" ): ignored_columns_by_pattern.append(col) + # We try to ignore nested columns as well + elif not self.config.profile_nested_fields and "." in col: + ignored_columns_by_pattern.append(col) elif col_dict.get("type") and self._should_ignore_column(col_dict["type"]): ignored_columns_by_type.append(col) else: @@ -407,9 +411,18 @@ def _get_columns_to_profile(self) -> List[str]: return columns_to_profile def _should_ignore_column(self, sqlalchemy_type: sa.types.TypeEngine) -> bool: - return str(sqlalchemy_type) in _get_column_types_to_ignore( - self.dataset.engine.dialect.name - ) + # We don't profiles columns with None types + if str(sqlalchemy_type) == "NULL": + return True + + sql_type = str(sqlalchemy_type) + + match = re.match(NORMALIZE_TYPE_PATTERN, sql_type) + + if match: + sql_type = match.group(1) + + return sql_type in _get_column_types_to_ignore(self.dataset.engine.dialect.name) @_run_with_query_combiner def _get_column_type(self, column_spec: _SingleColumnSpec, column: str) -> None: @@ -1397,6 +1410,8 @@ def _get_ge_dataset( def _get_column_types_to_ignore(dialect_name: str) -> List[str]: if dialect_name.lower() == POSTGRESQL: return ["JSON"] + elif dialect_name.lower() == BIGQUERY: + return ["ARRAY", "STRUCT", "GEOGRAPHY", "JSON"] return [] diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py index 9bf4451a188bcb..42d0def0a46e7d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_profiling_config.py @@ -188,6 +188,11 @@ class GEProfilingConfig(GEProfilingBaseConfig): ), ) + profile_nested_fields: bool = Field( + default=False, + description="Whether to profile complex types like structs, arrays and maps. ", + ) + @pydantic.root_validator(pre=True) def deprecate_bigquery_temp_table_schema(cls, values): # TODO: Update docs to remove mention of this field. From 07033a70158efc64cd7fd2da6e3f85b2014ae4e5 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 26 Nov 2024 10:07:39 -0600 Subject: [PATCH 073/174] docs(cloud): add documentation for Data Product side effect (#11948) --- docs/how/updating-datahub.md | 3 +++ docs/managed-datahub/release-notes/v_0_3_7.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 10173ccd4a1de4..087e30c2e541ad 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -88,6 +88,9 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Other Notable Changes - Downgrade to previous version is not automatically supported. +- Data Product Properties Unset side effect introduced + - Previously, Data Products could be set as linked to multiple Datasets if modified directly via the REST API rather than linked through the UI or GraphQL. This side effect aligns the REST API behavior with the GraphQL behavior by introducting a side effect that enforces the 1-to-1 constraint between Data Products and Datasets + - NOTE: There is a pathological pattern of writes for Data Products that can introduce issues with write processing that can occur with this side effect. If you are constantly changing all of the Datasets associated with a Data Product back and forth between multiple Data Products it will result in a high volume of writes due to the need to unset previous associations. ## 0.14.0.2 diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index a1514629768852..af23b5ae1541b5 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -120,3 +120,6 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies - (system / internal) Exclude form-prompt tests in live Metadata Tests evaluation - (system / internal) Exclude form-prompt tests in stored Metadata Test results - Elasticsearch reindex time limit of 8h removed + - Data Product Properties Unset side effect introduced + - Previously, Data Products could be set as linked to multiple Datasets if modified directly via the REST API rather than linked through the UI or GraphQL. This side effect aligns the REST API behavior with the GraphQL behavior by introducting a side effect that enforces the 1-to-1 constraint between Data Products and Datasets + - NOTE: There is a pathological pattern of writes for Data Products that can introduce issues with write processing that can occur with this side effect. If you are constantly changing all of the Datasets associated with a Data Product back and forth between multiple Data Products it will result in a high volume of writes due to the need to unset previous associations. From 2e3b4294bb63a916afcd7afc47c28cff5b7f176c Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Tue, 26 Nov 2024 21:46:42 +0530 Subject: [PATCH 074/174] feat(ingest/mssql): allow filtering by procedure_pattern (#11953) --- .../ingestion/source/sql/mssql/source.py | 22 ++++++-- .../golden_mces_mssql_no_db_with_filter.json | 54 ++----------------- .../source_files/mssql_no_db_with_filter.yml | 3 ++ 3 files changed, 26 insertions(+), 53 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py index 9ab9c76c30417f..7a2dbda8b4a939 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py @@ -50,6 +50,7 @@ BasicSQLAlchemyConfig, make_sqlalchemy_uri, ) +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.metadata.schema_classes import ( BooleanTypeClass, NumberTypeClass, @@ -78,6 +79,11 @@ class SQLServerConfig(BasicSQLAlchemyConfig): include_stored_procedures_code: bool = Field( default=True, description="Include information about object code." ) + procedure_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="Regex patterns for stored procedures to filter in ingestion." + "Specify regex to match the entire procedure name in database.schema.procedure_name format. e.g. to match all procedures starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'", + ) include_jobs: bool = Field( default=True, description="Include ingest of MSSQL Jobs. Requires access to the 'msdb' and 'sys' schema.", @@ -164,6 +170,8 @@ class SQLServerSource(SQLAlchemySource): If you do use pyodbc, make sure to change the source type from `mssql` to `mssql-odbc` so that we pull in the right set of dependencies. This will be needed in most cases where encryption is required, such as managed SQL Server services in Azure. """ + report: SQLSourceReport + def __init__(self, config: SQLServerConfig, ctx: PipelineContext): super().__init__(config, ctx, "mssql") # Cache the table and column descriptions @@ -416,10 +424,16 @@ def loop_stored_procedures( # noqa: C901 data_flow = MSSQLDataFlow(entity=mssql_default_job) with inspector.engine.connect() as conn: procedures_data_list = self._get_stored_procedures(conn, db_name, schema) - procedures = [ - StoredProcedure(flow=mssql_default_job, **procedure_data) - for procedure_data in procedures_data_list - ] + procedures: List[StoredProcedure] = [] + for procedure_data in procedures_data_list: + procedure_full_name = f"{db_name}.{schema}.{procedure_data['name']}" + if not self.config.procedure_pattern.allowed(procedure_full_name): + self.report.report_dropped(procedure_full_name) + continue + procedures.append( + StoredProcedure(flow=mssql_default_job, **procedure_data) + ) + if procedures: yield from self.construct_flow_workunits(data_flow=data_flow) for procedure in procedures: diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 3836e587ef8cf4..1d702214fedf79 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", + "job_id": "2a055367-5e6a-4162-b3a9-dd60f52c79a8", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-22 12:58:03.260000", - "date_modified": "2024-11-22 12:58:03.440000", + "date_created": "2024-11-26 07:22:19.640000", + "date_modified": "2024-11-26 07:22:19.773000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2282,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-22 12:58:03.137000", - "date_modified": "2024-11-22 12:58:03.137000" + "date_created": "2024-11-26 07:22:19.510000", + "date_modified": "2024-11-26 07:22:19.510000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2298,34 +2298,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", - "changeType": "UPSERT", - "aspectName": "dataJobInfo", - "aspect": { - "json": { - "customProperties": { - "procedure_depends_on": "{'DemoData.Foo.age_dist': 'USER_TABLE', 'DemoData.Foo.Items': 'USER_TABLE', 'DemoData.Foo.Persons': 'USER_TABLE', 'DemoData.Foo.SalesReason': 'USER_TABLE'}", - "depending_on_procedure": "{}", - "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", - "input parameters": "[]", - "date_created": "2024-11-22 12:58:03.140000", - "date_modified": "2024-11-22 12:58:03.140000" - }, - "externalUrl": "", - "name": "DemoData.Foo.NewProc", - "type": { - "string": "MSSQL_STORED_PROCEDURE" - } - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "container", "entityUrn": "urn:li:container:250ce23f940485303fa5e5d4f5194975", @@ -2713,22 +2685,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataJob", - "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),NewProc)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar)", diff --git a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml index 3749499074adfe..703f60b277b870 100644 --- a/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml +++ b/metadata-ingestion/tests/integration/sql_server/source_files/mssql_no_db_with_filter.yml @@ -9,6 +9,9 @@ source: database_pattern: deny: - NewData + procedure_pattern: + deny: + - DemoData.Foo.NewProc sink: type: file From 3b1a8ca926eba53b1cdbda85223d5c3e335bd6cd Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Tue, 26 Nov 2024 13:08:38 -0500 Subject: [PATCH 075/174] fix(test): updates a couple tests to disregard list order (#11840) Co-authored-by: nmbryant --- .../search/AggregateAcrossEntitiesResolverTest.java | 12 ++++++++++-- .../search/SearchAcrossEntitiesResolverTest.java | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java index 1b33118bd154af..0a8e4e8b4fa5f8 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolverTest.java @@ -386,7 +386,11 @@ private static EntityClient initMockEntityClient( Mockito.when( client.searchAcrossEntities( any(), - Mockito.eq(entityTypes), + Mockito.argThat( + argument -> + argument != null + && argument.containsAll(entityTypes) + && entityTypes.containsAll(argument)), Mockito.eq(query), Mockito.eq(filter), Mockito.eq(start), @@ -409,7 +413,11 @@ private static void verifyMockEntityClient( Mockito.verify(mockClient, Mockito.times(1)) .searchAcrossEntities( any(), - Mockito.eq(entityTypes), + Mockito.argThat( + argument -> + argument != null + && argument.containsAll(entityTypes) + && entityTypes.containsAll(argument)), Mockito.eq(query), Mockito.eq(filter), Mockito.eq(start), diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java index a601a815453b2f..42768b8a2de21b 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java @@ -462,7 +462,11 @@ private static EntityClient initMockEntityClient( Mockito.when( client.searchAcrossEntities( any(), - Mockito.eq(entityTypes), + Mockito.argThat( + argument -> + argument != null + && argument.containsAll(entityTypes) + && entityTypes.containsAll(argument)), Mockito.eq(query), Mockito.eq(filter), Mockito.eq(start), @@ -483,7 +487,11 @@ private static void verifyMockEntityClient( Mockito.verify(mockClient, Mockito.times(1)) .searchAcrossEntities( any(), - Mockito.eq(entityTypes), + Mockito.argThat( + argument -> + argument != null + && argument.containsAll(entityTypes) + && entityTypes.containsAll(argument)), Mockito.eq(query), Mockito.eq(filter), Mockito.eq(start), From 094433c3618983206dbbe17eb152b3db39a28820 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:57:47 -0600 Subject: [PATCH 076/174] fix(dataproduct): optimize data product sideeffect (#11961) --- .../DataProductUnsetSideEffect.java | 157 +++++++++++------- .../DataProductUnsetSideEffectTest.java | 112 ++++++++++++- .../SpringStandardPluginConfiguration.java | 3 +- 3 files changed, 201 insertions(+), 71 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java b/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java index 9c4bb52f014fc1..dae1a8ff51a2cf 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffect.java @@ -9,6 +9,7 @@ import com.linkedin.dataproduct.DataProductAssociation; import com.linkedin.dataproduct.DataProductAssociationArray; import com.linkedin.dataproduct.DataProductProperties; +import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.RetrieverContext; import com.linkedin.metadata.aspect.batch.ChangeMCP; import com.linkedin.metadata.aspect.batch.MCLItem; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import lombok.Getter; @@ -65,77 +67,108 @@ private static Stream generatePatchRemove( MCLItem mclItem, @Nonnull RetrieverContext retrieverContext) { if (DATA_PRODUCT_PROPERTIES_ASPECT_NAME.equals(mclItem.getAspectName())) { - List mcpItems = new ArrayList<>(); + DataProductProperties dataProductProperties = mclItem.getAspect(DataProductProperties.class); if (dataProductProperties == null) { log.error("Unable to process data product properties for urn: {}", mclItem.getUrn()); return Stream.empty(); } - Map> patchOpMap = new HashMap<>(); - for (DataProductAssociation dataProductAssociation : + DataProductAssociationArray newDataProductAssociationArray = Optional.ofNullable(dataProductProperties.getAssets()) - .orElse(new DataProductAssociationArray())) { - RelatedEntitiesScrollResult result = - retrieverContext - .getGraphRetriever() - .scrollRelatedEntities( - null, - QueryUtils.newFilter( - "urn", dataProductAssociation.getDestinationUrn().toString()), - null, - EMPTY_FILTER, - ImmutableList.of("DataProductContains"), - QueryUtils.newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), - Collections.emptyList(), - null, - 10, // Should only ever be one, if ever greater than ten will decrease over time - // to become consistent - null, - null); - if (!result.getEntities().isEmpty()) { - for (RelatedEntities entity : result.getEntities()) { - if (!mclItem.getUrn().equals(UrnUtils.getUrn(entity.getSourceUrn()))) { - GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp(); - patchOp.setOp(PatchOperationType.REMOVE.getValue()); - patchOp.setPath(String.format("/assets/%s", entity.getDestinationUrn())); - patchOpMap - .computeIfAbsent(entity.getSourceUrn(), urn -> new ArrayList<>()) - .add(patchOp); - } + .orElse(new DataProductAssociationArray()); + + DataProductProperties previousDataProductProperties = + mclItem.getPreviousAspect(DataProductProperties.class); + + if (!ChangeType.UPSERT.equals(mclItem.getChangeType()) + || previousDataProductProperties == null) { + // CREATE/CREATE_ENTITY/RESTATE + return generateUnsetMCPs(mclItem, newDataProductAssociationArray, retrieverContext); + } else { + // UPSERT with previous + DataProductAssociationArray oldDataProductAssociationArray = + Optional.ofNullable(previousDataProductProperties.getAssets()) + .orElse(new DataProductAssociationArray()); + + DataProductAssociationArray additions = + newDataProductAssociationArray.stream() + .filter(association -> !oldDataProductAssociationArray.contains(association)) + .collect(Collectors.toCollection(DataProductAssociationArray::new)); + + return generateUnsetMCPs(mclItem, additions, retrieverContext); + } + } + return Stream.empty(); + } + + private static Stream generateUnsetMCPs( + @Nonnull MCLItem dataProductItem, + @Nonnull DataProductAssociationArray dataProductAssociations, + @Nonnull RetrieverContext retrieverContext) { + List mcpItems = new ArrayList<>(); + Map> patchOpMap = new HashMap<>(); + + for (DataProductAssociation dataProductAssociation : dataProductAssociations) { + RelatedEntitiesScrollResult result = + retrieverContext + .getGraphRetriever() + .scrollRelatedEntities( + null, + QueryUtils.newFilter( + "urn", dataProductAssociation.getDestinationUrn().toString()), + null, + EMPTY_FILTER, + ImmutableList.of("DataProductContains"), + QueryUtils.newRelationshipFilter(EMPTY_FILTER, RelationshipDirection.INCOMING), + Collections.emptyList(), + null, + 10, // Should only ever be one, if ever greater than ten will decrease over time + // to become consistent + null, + null); + if (!result.getEntities().isEmpty()) { + for (RelatedEntities entity : result.getEntities()) { + if (!dataProductItem.getUrn().equals(UrnUtils.getUrn(entity.getSourceUrn()))) { + GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp(); + patchOp.setOp(PatchOperationType.REMOVE.getValue()); + patchOp.setPath(String.format("/assets/%s", entity.getDestinationUrn())); + patchOpMap + .computeIfAbsent(entity.getSourceUrn(), urn -> new ArrayList<>()) + .add(patchOp); } } } - for (String urn : patchOpMap.keySet()) { - EntitySpec entitySpec = - retrieverContext - .getAspectRetriever() - .getEntityRegistry() - .getEntitySpec(DATA_PRODUCT_ENTITY_NAME); - mcpItems.add( - PatchItemImpl.builder() - .urn(UrnUtils.getUrn(urn)) - .entitySpec( - retrieverContext - .getAspectRetriever() - .getEntityRegistry() - .getEntitySpec(DATA_PRODUCT_ENTITY_NAME)) - .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME) - .aspectSpec(entitySpec.getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys( - Map.of( - DataProductPropertiesTemplate.ASSETS_FIELD_NAME, - List.of(DataProductPropertiesTemplate.KEY_FIELD_NAME))) - .patch(patchOpMap.get(urn)) - .build() - .getJsonPatch()) - .auditStamp(mclItem.getAuditStamp()) - .systemMetadata(mclItem.getSystemMetadata()) - .build(retrieverContext.getAspectRetriever().getEntityRegistry())); - } - return mcpItems.stream(); } - return Stream.empty(); + for (String urn : patchOpMap.keySet()) { + EntitySpec entitySpec = + retrieverContext + .getAspectRetriever() + .getEntityRegistry() + .getEntitySpec(DATA_PRODUCT_ENTITY_NAME); + mcpItems.add( + PatchItemImpl.builder() + .urn(UrnUtils.getUrn(urn)) + .entitySpec( + retrieverContext + .getAspectRetriever() + .getEntityRegistry() + .getEntitySpec(DATA_PRODUCT_ENTITY_NAME)) + .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME) + .aspectSpec(entitySpec.getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys( + Map.of( + DataProductPropertiesTemplate.ASSETS_FIELD_NAME, + List.of(DataProductPropertiesTemplate.KEY_FIELD_NAME))) + .patch(patchOpMap.get(urn)) + .build() + .getJsonPatch()) + .auditStamp(dataProductItem.getAuditStamp()) + .systemMetadata(dataProductItem.getSystemMetadata()) + .build(retrieverContext.getAspectRetriever().getEntityRegistry())); + } + + return mcpItems.stream(); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java b/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java index 12dd57f94da23d..976b165fea53df 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/dataproducts/sideeffects/DataProductUnsetSideEffectTest.java @@ -17,6 +17,7 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.GraphRetriever; +import com.linkedin.metadata.aspect.SystemAspect; import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.models.graph.RelatedEntities; import com.linkedin.metadata.aspect.models.graph.RelatedEntitiesScrollResult; @@ -47,13 +48,7 @@ public class DataProductUnsetSideEffectTest { private static final EntityRegistry TEST_REGISTRY = new TestEntityRegistry(); private static final List SUPPORTED_CHANGE_TYPES = - List.of( - ChangeType.CREATE, - ChangeType.PATCH, - ChangeType.CREATE_ENTITY, - ChangeType.UPSERT, - ChangeType.DELETE, - ChangeType.RESTATE); + List.of(ChangeType.CREATE, ChangeType.CREATE_ENTITY, ChangeType.UPSERT, ChangeType.RESTATE); private static final Urn TEST_PRODUCT_URN = UrnUtils.getUrn("urn:li:dataProduct:someDataProductId"); @@ -358,6 +353,109 @@ public void testBulkAssetMove() { } } + @Test + public void testUpsertWithPreviousAspect() { + DataProductUnsetSideEffect test = new DataProductUnsetSideEffect(); + test.setConfig(TEST_PLUGIN_CONFIG); + + // Case 1: UPSERT with new additions + DataProductProperties previousProperties = new DataProductProperties(); + DataProductAssociationArray previousAssociations = new DataProductAssociationArray(); + DataProductAssociation previousAssociation = new DataProductAssociation(); + previousAssociation.setDestinationUrn(DATASET_URN_1); + previousAssociations.add(previousAssociation); + previousProperties.setAssets(previousAssociations); + + // New properties include both old and new datasets + DataProductProperties newProperties = new DataProductProperties(); + DataProductAssociationArray newAssociations = new DataProductAssociationArray(); + DataProductAssociation association1 = new DataProductAssociation(); + association1.setDestinationUrn(DATASET_URN_1); + DataProductAssociation association2 = new DataProductAssociation(); + association2.setDestinationUrn(DATASET_URN_2); + newAssociations.add(association1); + newAssociations.add(association2); + newProperties.setAssets(newAssociations); + + // Create change item with previous aspect + SystemAspect prevData = mock(SystemAspect.class); + when(prevData.getRecordTemplate()).thenReturn(previousProperties); + + ChangeItemImpl dataProductPropertiesChangeItem = + ChangeItemImpl.builder() + .urn(TEST_PRODUCT_URN) + .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME) + .changeType(ChangeType.UPSERT) + .entitySpec(TEST_REGISTRY.getEntitySpec(DATA_PRODUCT_ENTITY_NAME)) + .aspectSpec( + TEST_REGISTRY + .getEntitySpec(DATA_PRODUCT_ENTITY_NAME) + .getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)) + .recordTemplate(newProperties) + .previousSystemAspect(prevData) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(mockAspectRetriever); + + List testOutput = + test.postMCPSideEffect( + List.of( + MCLItemImpl.builder() + .build( + dataProductPropertiesChangeItem, + null, + null, + retrieverContext.getAspectRetriever())), + retrieverContext) + .toList(); + + // Verify that only one patch is generated for the new dataset + assertEquals( + testOutput.size(), 1, "Expected removal of previous data product for new dataset only"); + MCPItem patchItem = testOutput.get(0); + assertEquals( + patchItem.getUrn(), TEST_PRODUCT_URN_2, "Patch should target the old data product"); + GenericJsonPatch.PatchOp expectedPatchOp = new GenericJsonPatch.PatchOp(); + expectedPatchOp.setOp(PatchOperationType.REMOVE.getValue()); + expectedPatchOp.setPath(String.format("/assets/%s", DATASET_URN_2)); + + // Case 2: UPSERT with no new additions + DataProductProperties sameProperties = new DataProductProperties(); + DataProductAssociationArray sameAssociations = new DataProductAssociationArray(); + DataProductAssociation sameAssociation = new DataProductAssociation(); + sameAssociation.setDestinationUrn(DATASET_URN_1); + sameAssociations.add(sameAssociation); + sameProperties.setAssets(sameAssociations); + + SystemAspect prevSameData = mock(SystemAspect.class); + when(prevData.getRecordTemplate()).thenReturn(sameProperties); + + ChangeItemImpl noChangeItem = + ChangeItemImpl.builder() + .urn(TEST_PRODUCT_URN) + .aspectName(DATA_PRODUCT_PROPERTIES_ASPECT_NAME) + .changeType(ChangeType.UPSERT) + .entitySpec(TEST_REGISTRY.getEntitySpec(DATA_PRODUCT_ENTITY_NAME)) + .aspectSpec( + TEST_REGISTRY + .getEntitySpec(DATA_PRODUCT_ENTITY_NAME) + .getAspectSpec(DATA_PRODUCT_PROPERTIES_ASPECT_NAME)) + .recordTemplate(sameProperties) + .previousSystemAspect(prevSameData) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(mockAspectRetriever); + + List noChangeOutput = + test.postMCPSideEffect( + List.of( + MCLItemImpl.builder() + .build(noChangeItem, null, null, retrieverContext.getAspectRetriever())), + retrieverContext) + .toList(); + + // Verify no patches are generated when there are no new additions + assertEquals(noChangeOutput.size(), 0, "Expected no changes when assets are the same"); + } + private static DataProductProperties getTestDataProductProperties(Urn destinationUrn) { DataProductProperties dataProductProperties = new DataProductProperties(); DataProductAssociationArray dataProductAssociations = new DataProductAssociationArray(); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java index b2db0857a6a5c8..26e0da8e6fb990 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/plugins/SpringStandardPluginConfiguration.java @@ -98,8 +98,7 @@ public MCPSideEffect dataProductUnsetSideEffect() { AspectPluginConfig.builder() .enabled(true) .className(DataProductUnsetSideEffect.class.getName()) - .supportedOperations( - List.of("CREATE", "CREATE_ENTITY", "UPSERT", "RESTATE", "DELETE", "PATCH")) + .supportedOperations(List.of("CREATE", "CREATE_ENTITY", "UPSERT", "RESTATE")) .supportedEntityAspectNames( List.of( AspectPluginConfig.EntityAspectName.builder() From 1ba1b2c146646a5f1541632091a4a182e2c41608 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Tue, 26 Nov 2024 22:15:33 +0100 Subject: [PATCH 077/174] fix(gms/patch): Fix Upstream lineage patching when path contained encoded slash (#11957) --- .../aspect/patch/template/TemplateUtil.java | 5 ++-- .../template/UpstreamLineageTemplateTest.java | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java index d4e94e1e82e8f6..2423e37e6d5419 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/TemplateUtil.java @@ -81,14 +81,13 @@ public static JsonNode populateTopLevelKeys(JsonNode transformedNode, JsonPatch PatchOperationType.REMOVE.equals(operationPath.getFirst()) ? keys.length : keys.length - 1; - // Skip first as it will always be blank due to path starting with / for (int i = 1; i < endIdx; i++) { + String decodedKey = decodeValue(keys[i]); if (parent.get(keys[i]) == null) { - String decodedKey = decodeValue(keys[i]); ((ObjectNode) parent).set(decodedKey, instance.objectNode()); } - parent = parent.get(keys[i]); + parent = parent.get(decodedKey); } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java index 5042c35d2f5d47..f9af15a3d4cc6c 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java @@ -185,6 +185,29 @@ public void testPatchUpstream() throws Exception { // New entry in array because of new transformation type assertEquals(result4.getFineGrainedLineages().get(3), fineGrainedLineage4); + JsonPatchBuilder patchOperations5 = Json.createPatchBuilder(); + String urn4 = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,test-bucket/hive/folder_1/folder_2/my_dataset,DEV),c2)"; + UrnArray downstreamUrns5 = new UrnArray(); + downstreamUrns5.add(Urn.createFromString(urn4)); + patchOperations5.add( + "/fineGrainedLineages/TRANSFORM/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,test-bucket~1hive~1folder_1~1folder_2~1my_dataset,DEV),c2)/urn:li:query:anotherQuery/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:bigquery,upstream_table_1,PROD),c2)", + finegrainedLineageNode5.build()); + JsonPatch jsonPatch5 = patchOperations5.build(); + UpstreamLineage result5 = upstreamLineageTemplate.applyPatch(result4, jsonPatch5); + // Hack because Jackson parses values to doubles instead of floats + DataMap dataMap5 = new DataMap(); + dataMap5.put("confidenceScore", 1.0); + FineGrainedLineage fineGrainedLineage5 = new FineGrainedLineage(dataMap5); + fineGrainedLineage5.setUpstreams(upstreamUrns3); + fineGrainedLineage5.setDownstreams(downstreamUrns5); + fineGrainedLineage5.setTransformOperation("TRANSFORM"); + fineGrainedLineage5.setUpstreamType(FineGrainedLineageUpstreamType.FIELD_SET); + fineGrainedLineage5.setDownstreamType(FineGrainedLineageDownstreamType.FIELD); + fineGrainedLineage5.setQuery(UrnUtils.getUrn("urn:li:query:anotherQuery")); + // New entry in array because of new transformation type + assertEquals(result5.getFineGrainedLineages().get(4), fineGrainedLineage5); + // Remove JsonPatchBuilder removeOperations = Json.createPatchBuilder(); removeOperations.remove( From fbc98518b6285d61563723ec112050115602eb13 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 27 Nov 2024 03:19:14 +0530 Subject: [PATCH 078/174] fix(ingest): always send correct data for advanced section (#11960) --- .../src/app/ingest/source/IngestionSourceList.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx index ccfa200fab630f..1990a3d7798973 100644 --- a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx +++ b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx @@ -193,7 +193,9 @@ export const IngestionSourceList = () => { const formatExtraArgs = (extraArgs): StringMapEntryInput[] => { if (extraArgs === null || extraArgs === undefined) return []; - return extraArgs.map((entry) => ({ key: entry.key, value: entry.value })); + return extraArgs + .filter((entry) => entry.value !== null && entry.value !== undefined && entry.value !== '') + .map((entry) => ({ key: entry.key, value: entry.value })); }; const createOrUpdateIngestionSource = ( From f9cc67dd31ff23431e3e7e7dbe4ae8f12d7657b7 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Tue, 26 Nov 2024 14:03:22 -0800 Subject: [PATCH 079/174] feat(schematron): add java capabilities for schema translation (#11963) --- build.gradle | 1 + docs-website/sidebars.js | 1 + .../java/datahub-client/build.gradle | 3 +- .../java/datahub-client/scripts/check_jar.sh | 5 +- .../java/datahub-schematron/README.md | 73 +++ .../java/datahub-schematron/cli/build.gradle | 110 ++++ .../cli/scripts/avro_schema_to_mce.py | 94 +++ .../cli/scripts/mce_diff.py | 345 ++++++++++ .../schematron/cli/SchemaTron.java | 147 +++++ .../schematron/SchemaTranslatorTest.java | 149 +++++ .../src/test/resources/CustomerProfile.avsc | 456 +++++++++++++ .../src/test/resources/CustomerProfile2.avsc | 244 +++++++ .../cli/src/test/resources/FlatUser.avsc | 45 ++ .../diffs/CustomerProfile2_diff.json | 125 ++++ .../resources/diffs/CustomerProfile_diff.json | 181 ++++++ .../java/datahub-schematron/lib/build.gradle | 130 ++++ .../converters/SchemaConverter.java | 25 + .../converters/avro/AvroSchemaConverter.java | 607 ++++++++++++++++++ .../schematron/models/DataHubType.java | 40 ++ .../schematron/models/FieldElement.java | 38 ++ .../schematron/models/FieldPath.java | 173 +++++ .../schematron/utils/Constants.java | 12 + .../schematron/models/FieldPathTest.java | 246 +++++++ .../src/test/resources/CustomerProfile.avsc | 456 +++++++++++++ .../src/test/resources/CustomerProfile2.avsc | 244 +++++++ .../lib/src/test/resources/FlatUser.avsc | 45 ++ .../spark-lineage-legacy/scripts/check_jar.sh | 4 +- settings.gradle | 2 + 28 files changed, 3998 insertions(+), 3 deletions(-) create mode 100644 metadata-integration/java/datahub-schematron/README.md create mode 100644 metadata-integration/java/datahub-schematron/cli/build.gradle create mode 100644 metadata-integration/java/datahub-schematron/cli/scripts/avro_schema_to_mce.py create mode 100644 metadata-integration/java/datahub-schematron/cli/scripts/mce_diff.py create mode 100644 metadata-integration/java/datahub-schematron/cli/src/main/java/io/datahubproject/schematron/cli/SchemaTron.java create mode 100644 metadata-integration/java/datahub-schematron/cli/src/test/java/io/datahubproject/schematron/SchemaTranslatorTest.java create mode 100644 metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile.avsc create mode 100644 metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile2.avsc create mode 100644 metadata-integration/java/datahub-schematron/cli/src/test/resources/FlatUser.avsc create mode 100644 metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile2_diff.json create mode 100644 metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile_diff.json create mode 100644 metadata-integration/java/datahub-schematron/lib/build.gradle create mode 100644 metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/SchemaConverter.java create mode 100644 metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java create mode 100644 metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/DataHubType.java create mode 100644 metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldElement.java create mode 100644 metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java create mode 100644 metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/utils/Constants.java create mode 100644 metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/models/FieldPathTest.java create mode 100644 metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile.avsc create mode 100644 metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile2.avsc create mode 100644 metadata-integration/java/datahub-schematron/lib/src/test/resources/FlatUser.avsc diff --git a/build.gradle b/build.gradle index 9ee756d41e11ef..e3c4f5efe6bb63 100644 --- a/build.gradle +++ b/build.gradle @@ -350,6 +350,7 @@ allprojects { } } } + } configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) { diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 0470723c1adb79..3a9d6e10ea8d42 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -989,6 +989,7 @@ module.exports = { // "metadata-ingestion/examples/structured_properties/README" // "smoke-test/tests/openapi/README" // "docs/SECURITY_STANCE" + // "metadata-integration/java/datahub-schematron/README" // ], ], }; diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 1bdc848d0385b1..56a486ad043305 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -19,6 +19,7 @@ jar { dependencies { api project(':entity-registry') api project(':metadata-integration:java:datahub-event') + implementation project(':metadata-integration:java:datahub-schematron:lib') implementation(externalDependency.kafkaAvroSerializer) { exclude group: "org.apache.avro" } @@ -114,7 +115,7 @@ shadowJar { relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework' relocate 'com.google.errorprone', 'datahub.shaded.com.google.errorprone' // Below jars added for kafka emitter only - relocate 'org.apache.avro', 'datahub.shaded.org.apache.avro' +// relocate 'org.apache.avro', 'datahub.shaded.org.apache.avro' relocate 'com.thoughtworks.paranamer', 'datahub.shaded.com.thoughtworks.paranamer' relocate 'org.xerial.snappy', 'datahub.shaded.org.xerial.snappy' relocate 'org.apache.kafka', 'datahub.shaded.org.apache.kafka' diff --git a/metadata-integration/java/datahub-client/scripts/check_jar.sh b/metadata-integration/java/datahub-client/scripts/check_jar.sh index 10299ec714d165..e451a7dd2a009e 100755 --- a/metadata-integration/java/datahub-client/scripts/check_jar.sh +++ b/metadata-integration/java/datahub-client/scripts/check_jar.sh @@ -40,7 +40,10 @@ jar -tvf $jarFile |\ grep -v "mozilla" |\ grep -v "VersionInfo.java" |\ grep -v "mime.types" |\ - grep -v "com/ibm/.*" + grep -v "com/ibm/.*" |\ + grep -v "org/apache/avro" |\ + grep -v "org/apache" + if [ $? -ne 0 ]; then diff --git a/metadata-integration/java/datahub-schematron/README.md b/metadata-integration/java/datahub-schematron/README.md new file mode 100644 index 00000000000000..0dc1c2b9c74551 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/README.md @@ -0,0 +1,73 @@ +# SchemaTron (Incubating) + +> ⚠️ This is an incubating project in draft status. APIs and functionality may change significantly between releases. + +SchemaTron is a schema translation toolkit that converts between various schema formats and DataHub's native schema representation. It currently provides robust support for Apache Avro schema translation with a focus on complex schema structures including unions, arrays, maps, and nested records. + +## Modules + +### CLI Module + +Command-line interface for converting schemas and emitting them to DataHub. + +```bash +# Execute from this directory +../../../gradlew :metadata-integration:java:datahub-schematron:cli:run --args="-i cli/src/test/resources/FlatUser.avsc" +``` + +#### CLI Options + +- `-i, --input`: Input schema file or directory path +- `-p, --platform`: Data platform name (default: "avro") +- `-s, --server`: DataHub server URL (default: "http://localhost:8080") +- `-t, --token`: DataHub access token +- `--sink`: Output sink - "rest" or "file" (default: "rest") +- `--output-file`: Output file path when using file sink (default: "metadata.json") + +### Library Module + +Core translation logic and models for schema conversion. Features include: + +- Support for complex Avro schema structures: + - Union types with multiple record options + - Nested records and arrays + - Optional fields with defaults + - Logical types (date, timestamp, etc.) + - Maps with various value types + - Enum types + - Custom metadata and documentation + +- Comprehensive path handling for schema fields +- DataHub-compatible metadata generation +- Schema fingerprinting and versioning + +## Example Schema Support + +The library can handle sophisticated schema structures including: + +- Customer profiles with multiple identification types (passport, driver's license, national ID) +- Contact information with primary and alternative contact methods +- Address validation with verification metadata +- Subscription history tracking +- Flexible preference and metadata storage +- Tagged customer attributes + +## Development + +The project includes extensive test coverage through: + +- Unit tests for field path handling +- Schema translation comparison tests +- Integration tests with Python reference implementation + +Test resources include example schemas demonstrating various Avro schema features and edge cases. + +## Contributing + +As this is an incubating project, we welcome contributions and feedback on: + +- Additional schema format support +- Improved handling of complex schema patterns +- Enhanced metadata translation +- Documentation and examples +- Test coverage \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/cli/build.gradle b/metadata-integration/java/datahub-schematron/cli/build.gradle new file mode 100644 index 00000000000000..1711ff947c2d19 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/build.gradle @@ -0,0 +1,110 @@ +plugins { + id "application" +} +apply plugin: 'java' +apply plugin: 'jacoco' + +ext { + javaMainClass = "io.datahubproject.schematron.cli.SchemaTron" +} + +application { + mainClassName = javaMainClass +} + +dependencies { + // Existing dependencies remain unchanged + implementation 'info.picocli:picocli:4.7.5' + annotationProcessor 'info.picocli:picocli-codegen:4.7.5' + implementation 'ch.qos.logback:logback-classic:1.2.11' + implementation 'ch.qos.logback:logback-core:1.2.11' + implementation project(':metadata-integration:java:datahub-client') + implementation project(':metadata-integration:java:datahub-schematron:lib') + implementation externalDependency.avro + compileOnly externalDependency.lombok + annotationProcessor externalDependency.lombok + + // Test dependencies + testImplementation externalDependency.testng + testImplementation externalDependency.mockito +} + +test { + useTestNG() + + testLogging { + events "passed", "skipped", "failed" + exceptionFormat "full" + showStandardStreams = true + } + + systemProperty 'python.venv.path', System.getProperty('python.venv.path', '../venv') +} + +task validatePythonEnv { + doFirst { + def venvPath = System.getProperty('python.venv.path', '../../../../metadata-ingestion/venv') + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def pythonExe = isWindows ? "${venvPath}/Scripts/python.exe" : "${venvPath}/bin/python" + + def result = exec { + commandLine pythonExe, "-c", "import sys; print(sys.executable)" + ignoreExitValue = true + standardOutput = new ByteArrayOutputStream() + errorOutput = new ByteArrayOutputStream() + } + + if (result.exitValue != 0) { + throw new GradleException("Python virtual environment not properly set up at ${venvPath}") + } + } +} + +test.dependsOn tasks.getByPath(":metadata-ingestion:installDev") + +jacocoTestReport { + dependsOn test +} + +test.finalizedBy jacocoTestReport + +task updateGoldenFiles { + dependsOn validatePythonEnv + doLast { + def venvPath = System.getProperty('python.venv.path', '../../../../metadata-ingestion/venv') + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def pythonExe = isWindows ? "${venvPath}/Scripts/python.exe" : "${venvPath}/bin/python" + def diffsDir = new File('src/test/resources/diffs') + + if (!diffsDir.exists()) { + throw new GradleException("Diffs directory not found at ${diffsDir.absolutePath}") + } + + // Find all json files in the diffs directory + diffsDir.listFiles().findAll { it.name.endsWith('_diff.json') }.each { diffFile -> + def baseName = diffFile.name.replace('_diff.json', '') + def pythonOutput = "build/test-outputs/${baseName}_python.json" + def javaOutput = "build/test-outputs/${baseName}_java.json" + + println "Updating golden file for ${baseName}..." + + exec { + commandLine pythonExe, + 'scripts/mce_diff.py', + '--update-golden-diff', + '--golden-diff-file', + diffFile.absolutePath, + pythonOutput, + javaOutput + ignoreExitValue = true + standardOutput = new ByteArrayOutputStream() + errorOutput = new ByteArrayOutputStream() + } + } + } +} + +configurations { + provided + implementation.extendsFrom provided +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/cli/scripts/avro_schema_to_mce.py b/metadata-integration/java/datahub-schematron/cli/scripts/avro_schema_to_mce.py new file mode 100644 index 00000000000000..38a90bc3318428 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/scripts/avro_schema_to_mce.py @@ -0,0 +1,94 @@ +from datahub.ingestion.extractor.schema_util import AvroToMceSchemaConverter +from avro.schema import parse as parse_avro, RecordSchema +from datahub.emitter.synchronized_file_emitter import SynchronizedFileEmitter +import datahub.metadata.schema_classes as models +import click +from datahub.emitter.mce_builder import make_data_platform_urn, make_dataset_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +import os +import hashlib +from datahub.ingestion.graph.client import get_default_graph + + +def get_schema_hash(schema): + # Convert schema to string if it isn't already + schema_str = str(schema) + + # Create MD5 hash + schema_hash = hashlib.md5(schema_str.encode("utf-8")).hexdigest() + + return schema_hash + + +@click.command(name="avro2datahub") +@click.option("--input-file", "-i", type=click.Path(exists=True), required=True) +@click.option("--platform", type=str, required=True) +@click.option("--output-file", "-o", type=click.Path(), default="metadata.py.json") +@click.option("--to-file", "-f", is_flag=True, default=True) +@click.option("--to-server", "-s", is_flag=True, default=False) +def generate_schema_file_from_avro_schema( + input_file: str, platform: str, output_file: str, to_file: bool, to_server: bool +): + avro_schema_file = input_file + output_file_name = output_file + platform_urn = make_data_platform_urn(platform) + converter = AvroToMceSchemaConverter(is_key_schema=False) + + # Delete the output file if it exists + if os.path.exists(output_file_name): + os.remove(output_file_name) + + with open(avro_schema_file) as f: + raw_string = f.read() + avro_schema = parse_avro(raw_string) + # Get fingerprint bytes + canonical_form = avro_schema.canonical_form + print( + f"Schema canonical form: Length ({len(canonical_form)}); {canonical_form}" + ) + md5_bytes = avro_schema.fingerprint("md5") + # Convert to hex string + avro_schema_hash = md5_bytes.hex() + assert isinstance( + avro_schema, RecordSchema + ), "This command only works for Avro records" + dataset_urn = make_dataset_urn( + platform=platform_urn, + name=( + f"{avro_schema.namespace}.{avro_schema.name}" + if avro_schema.namespace + else avro_schema.name + ), + ) + schema_fields = [ + f for f in converter.to_mce_fields(avro_schema, is_key_schema=False) + ] + schema_metadata = models.SchemaMetadataClass( + schemaName=avro_schema.name, + platform=platform_urn, + version=0, + hash=avro_schema_hash, + platformSchema=models.OtherSchemaClass(rawSchema=raw_string), + fields=schema_fields, + ) + assert schema_metadata.validate() + if to_file: + with SynchronizedFileEmitter(output_file_name) as file_emitter: + file_emitter.emit( + MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=schema_metadata + ) + ) + if to_server: + with get_default_graph() as graph: + graph.emit( + MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=schema_metadata + ) + ) + + print(f"Wrote metadata to {output_file}") + + +if __name__ == "__main__": + generate_schema_file_from_avro_schema() diff --git a/metadata-integration/java/datahub-schematron/cli/scripts/mce_diff.py b/metadata-integration/java/datahub-schematron/cli/scripts/mce_diff.py new file mode 100644 index 00000000000000..37ba11138610c1 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/scripts/mce_diff.py @@ -0,0 +1,345 @@ +import json +from typing import Dict, Any, Optional, Tuple +import click + +import json +from typing import Dict, Any + + +def diff_lists(list1, list2): + """ + Compare two lists element by element and return their differences. + + Args: + list1 (list): First list to compare + list2 (list): Second list to compare + + Returns: + dict: A dictionary containing the differences + """ + result = {"added": {}, "removed": {}, "modified": set(), "modified_details": {}} + + if len(list1) != len(list2): + # Let's first line up the elements that are common to both lists using + # the fieldPath as the key if it exists + if "fieldPath" in list1[0]: + list1_dict = {field["fieldPath"]: field for field in list1} + list2_dict = {field["fieldPath"]: field for field in list2} + common_keys = set(list1_dict.keys()) & set(list2_dict.keys()) + list1 = [list1_dict[key] for key in common_keys] + list2 = [list2_dict[key] for key in common_keys] + list1.extend( + [list1_dict[key] for key in set(list1_dict.keys()) - common_keys] + ) + list2.extend( + [list2_dict[key] for key in set(list2_dict.keys()) - common_keys] + ) + + # Handle added elements (if list2 is longer) + if len(list2) > len(list1): + for i in range(len(list1), len(list2)): + if "fieldPath" in list2[i]: + result["added"][list2[i]["fieldPath"]] = list2[i] + else: + result["added"][str(i)] = list2[i] + + # Handle removed elements (if list1 is longer) + if len(list1) > len(list2): + for i in range(len(list2), len(list1)): + if "fieldPath" in list1[i]: + result["removed"][list1[i]["fieldPath"]] = list1[i] + else: + result["removed"][str(i)] = list1[i] + + # Compare common indices + for i in range(min(len(list1), len(list2))): + value1 = list1[i] + value2 = list2[i] + + if type(value1) != type(value2): + result["modified"].add(str(i)) + result["modified_details"][str(i)] = {"before": value1, "after": value2} + elif isinstance(value1, dict) and isinstance(value2, dict): + nested_diff = diff_dicts( + value1, value2, identifier=value1.get("fieldPath", i) + ) + if any(nested_diff.values()): + result["modified"].add(value1.get("fieldPath", i)) + result["modified_details"][value1.get("fieldPath", i)] = nested_diff + elif isinstance(value1, list) and isinstance(value2, list): + nested_diff = diff_lists(value1, value2) + if any(nested_diff.values()): + result["modified"].add(str(i)) + result["modified_details"][str(i)] = nested_diff + elif value1 != value2: + result["modified"].add(str(i)) + result["modified_details"][str(i)] = { + "before": value1, + "after": value2, + "identifier": i, + } + + return result + + +def diff_schema_field(field1_dict, field2_dict): + + from datahub.metadata.schema_classes import SchemaFieldClass + + field1 = SchemaFieldClass.from_obj(field1_dict) + field2 = SchemaFieldClass.from_obj(field2_dict) + + # Initialize result structure + result = {"added": {}, "removed": {}, "modified": set(), "modified_details": {}} + + result = {} + if field1.fieldPath != field2.fieldPath: + result["fieldPath"] = {"before": field1.fieldPath, "after": field2.fieldPath} + + if field1.type != field2.type: + result["type"] = { + "before": field1.type, + "after": field2.type, + "identifier": field1.fieldPath, + } + + if field1.description != field2.description: + result["description"] = { + "before": field1.description, + "after": field2.description, + "identifier": field1.fieldPath, + } + + if field1.nullable != field2.nullable: + result["nullable"] = { + "before": field1.nullable, + "after": field2.nullable, + "identifier": field1.fieldPath, + } + + return result + + +def diff_schema_metadata(schema1_dict, schema2_dict): + + ignored_for_diff = [ + "created", + "modified", + "hash", + "platformSchema", + "lastModified", + ] # TODO: Reduce this list + + for key in ignored_for_diff: + schema1_dict.pop(key, None) + schema2_dict.pop(key, None) + + return diff_dicts(schema1_dict, schema2_dict) + + +def is_empty_diff(diff_dict) -> bool: + if diff_dict.keys() == EMPTY_DIFF().keys(): + for key in diff_dict: + if diff_dict[key]: + return False + return True + return False + + +def format_diff(diff_dict) -> Any: + if isinstance(diff_dict, set): + diff_dict = sorted(list([x for x in diff_dict])) + elif isinstance(diff_dict, dict): + for key in diff_dict: + diff_dict[key] = format_diff(diff_dict[key]) + return diff_dict + + +def EMPTY_DIFF(): + return { + "added": {}, + "removed": {}, + "modified": set(), + "modified_details": {}, + } + + +def diff_dicts(dict1, dict2, identifier=None): + """ + Compare two dictionaries recursively and return their differences. + + Args: + dict1 (dict): First dictionary to compare + dict2 (dict): Second dictionary to compare + + Returns: + dict: A dictionary containing the differences with the following structure: + { + 'added': Keys present in dict2 but not in dict1, + 'removed': Keys present in dict1 but not in dict2, + 'modified': Keys present in both but with different values, + 'modified_details': Detailed before/after values for modified keys + } + """ + if "nullable" in dict1: + # Assume this is a SchemaFieldClass + return diff_schema_field(dict1, dict2) + + if "hash" in dict1: + # Assume this is a schema metadata class + return diff_schema_metadata(dict1, dict2) + + dict1_keys = set(dict1.keys()) + dict2_keys = set(dict2.keys()) + + # Find keys that were added, removed, or modified + added_keys = dict2_keys - dict1_keys + removed_keys = dict1_keys - dict2_keys + common_keys = dict1_keys & dict2_keys + + # Initialize result structure + result = EMPTY_DIFF() + # Handle added keys + for key in added_keys: + result["added"][key] = dict2[key] + + # Handle removed keys + for key in removed_keys: + result["removed"][key] = dict1[key] + + # Check common keys for modifications + for key in common_keys: + value1 = dict1[key] + value2 = dict2[key] + + # If both values are dictionaries, recurse + if isinstance(value1, dict) and isinstance(value2, dict): + nested_diff = diff_dicts( + value1, value2, identifier=value1.get("fieldPath", key) + ) + if any(nested_diff.values()): # If there are any differences + result["modified"].add(key) + result["modified_details"][key] = nested_diff + # If both values are lists, compare them element by element + elif isinstance(value1, list) and isinstance(value2, list): + nested_diff = diff_lists(value1, value2) + if any(nested_diff.values()): + result["modified"].add(key) + result["modified_details"][key] = nested_diff + # Otherwise compare directly + elif value1 != value2: + result["modified"].add(key) + result["modified_details"][key] = { + "before": value1, + "after": value2, + "identifier": identifier, + } + + return result + + +def process_single_element(element) -> Tuple[str, str, Dict[str, Any]]: + if "entityUrn" in element: + entity = element["entityUrn"] + else: + raise Exception("Element does not have an entityUrn key") + if "aspectName" in element: + aspect = element["aspectName"] + else: + raise Exception("Element does not have an aspectName key") + if "aspect" in element: + if "json" in element["aspect"]: + return entity, aspect, element["aspect"]["json"] + elif "value" in element["aspect"]: + json_value = json.loads(element["aspect"]["value"]) + return entity, aspect, json_value + else: + raise Exception("Element does not have a json or value key") + else: + raise Exception("Element does not have an aspect key") + + +def process_element_with_dict(element, global_dict): + entity, aspect, data = process_single_element(element) + if entity not in global_dict: + global_dict[entity] = {} + if aspect not in global_dict[entity]: + global_dict[entity][aspect] = data + else: + # breakpoint() + raise Exception("Duplicate aspect found") + + +@click.command("compute_diff") +@click.argument("input_file_1", type=click.Path(exists=True)) +@click.argument("input_file_2", type=click.Path(exists=True)) +@click.option("--golden-diff-file", type=click.Path(), default=None) +@click.option("--update-golden-diff", is_flag=True) +def compute_diff( + input_file_1: str, + input_file_2: str, + golden_diff_file: Optional[str] = None, + update_golden_diff: bool = False, +): + + # Read the files into json objects and compare them + # If they are the same, exit 0 + # If they are different, exit 1 + file_1_mcps = {} + with open(input_file_1) as file1: + data1 = json.load(file1) + assert isinstance(data1, list) + for element in data1: + process_element_with_dict(element, file_1_mcps) + print(f"Processed {len(file_1_mcps)} elements from file {input_file_1}") + + file_2_mcps = {} + with open(input_file_2) as file2: + data2 = json.load(file2) + assert isinstance(data2, list) + for element in data2: + process_element_with_dict(element, file_2_mcps) + + print(f"Processed {len(file_2_mcps)} elements from file {input_file_2}") + + if golden_diff_file and not update_golden_diff: + with open(golden_diff_file) as golden_diff: + golden_diff_data = json.load(golden_diff) + else: + golden_diff_data = None + + computed_diff_data = {} + + assert len(file_1_mcps) == len(file_2_mcps) + for entity in file_1_mcps: + assert entity in file_2_mcps + assert len(file_1_mcps[entity]) == len(file_2_mcps[entity]) + for aspect in file_1_mcps[entity]: + assert aspect in file_2_mcps[entity] + aspect_diff = diff_dicts( + file_1_mcps[entity][aspect], file_2_mcps[entity][aspect] + ) + if golden_diff_data: + assert aspect in golden_diff_data[entity] + assert format_diff(aspect_diff) == golden_diff_data[entity][aspect], ( + f"Computed difference is {json.dumps(format_diff(aspect_diff), indent=2)}\n" + f"Expected difference is {json.dumps(golden_diff_data[entity][aspect], indent=2)}" + ) + + else: + if update_golden_diff: + if entity not in computed_diff_data: + computed_diff_data[entity] = {} + computed_diff_data[entity][aspect] = format_diff(aspect_diff) + else: + assert is_empty_diff( + aspect_diff + ), f"Difference is {json.dumps(format_diff(aspect_diff), indent=2)}" + + if update_golden_diff: + with open(golden_diff_file, "w") as golden_diff: + json.dump(computed_diff_data, golden_diff, indent=2, sort_keys=True) + + +if __name__ == "__main__": + compute_diff() diff --git a/metadata-integration/java/datahub-schematron/cli/src/main/java/io/datahubproject/schematron/cli/SchemaTron.java b/metadata-integration/java/datahub-schematron/cli/src/main/java/io/datahubproject/schematron/cli/SchemaTron.java new file mode 100644 index 00000000000000..d8e4a43cfa8fba --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/src/main/java/io/datahubproject/schematron/cli/SchemaTron.java @@ -0,0 +1,147 @@ +package io.datahubproject.schematron.cli; + +import com.linkedin.common.FabricType; +import com.linkedin.common.urn.DataPlatformUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.schema.SchemaField; +import com.linkedin.schema.SchemaMetadata; +import datahub.client.Emitter; +import datahub.client.file.FileEmitter; +import datahub.client.file.FileEmitterConfig; +import datahub.client.rest.RestEmitter; +import datahub.event.MetadataChangeProposalWrapper; +import io.datahubproject.schematron.converters.avro.AvroSchemaConverter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.Callable; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.Schema; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Slf4j +@Command( + name = "schema-translator", + description = "Converts schemas to DataHub format and emits them", + mixinStandardHelpOptions = true) +public class SchemaTron implements Callable { + + @Option( + names = {"-i", "--input"}, + description = "Input schema file or directory") + private String input; + + @Option( + names = {"-s", "--server"}, + description = "DataHub server URL", + required = false, + defaultValue = "http://localhost:8080") + private String server; + + @Option( + names = {"-t", "--token"}, + description = "DataHub access token", + required = false, + defaultValue = "") + private String token; + + @Option( + names = {"-p", "--platform"}, + description = "Data platform name", + defaultValue = "avro") + private String platform; + + @Option( + names = {"--sink"}, + description = "DataHub sink name", + defaultValue = "rest") + private String sink; + + @Option( + names = {"--output-file"}, + description = "Output file for the emitted metadata", + defaultValue = "metadata.json") + private String outputFile; + + private final AvroSchemaConverter schemaConverter = AvroSchemaConverter.builder().build(); + + @Override + public Integer call() throws Exception { + + Emitter emitter; + if (sink.equals("rest")) { + emitter = RestEmitter.create(b -> b.server(server).token(token)); + } else if (sink.equals("file")) { + emitter = new FileEmitter(FileEmitterConfig.builder().fileName(outputFile).build()); + } else { + throw new IllegalArgumentException("Unsupported sink: " + sink); + } + + try { + // Process input files + Stream inputFiles; + Path inputPath = Path.of(input); + if (Files.isDirectory(inputPath)) { + inputFiles = Files.walk(inputPath).filter(p -> p.toString().endsWith(".avsc")); + } else { + inputFiles = Stream.of(inputPath); + } + + // Process each file + inputFiles.forEach( + filePath -> { + try { + // Read and parse Avro schema + String schemaStr = Files.readString(filePath); + Schema avroSchema = new Schema.Parser().parse(schemaStr); + + // Convert to DataHub schema + boolean isKeySchema = false; + boolean isDefaultNullable = false; + SchemaMetadata schemaMetadata = + schemaConverter.toDataHubSchema( + avroSchema, + isKeySchema, + isDefaultNullable, + new DataPlatformUrn(platform), + null); + log.info("Generated {} fields", schemaMetadata.getFields().size()); + for (SchemaField field : schemaMetadata.getFields()) { + log.debug("Field path: {}", field.getFieldPath()); + } + + DatasetUrn datasetUrn = + new DatasetUrn( + new DataPlatformUrn(platform), avroSchema.getFullName(), FabricType.PROD); + + MetadataChangeProposalWrapper wrapper = + new MetadataChangeProposalWrapper( + "dataset", + datasetUrn.toString(), + ChangeType.UPSERT, + schemaMetadata, + "schemaMetadata"); + + // Emit to DataHub + emitter.emit(wrapper, null).get(); + log.info("Emitted schema for {}", datasetUrn); + } catch (Exception e) { + System.err.println("Error processing file: " + filePath); + e.printStackTrace(); + } + }); + + return 0; + } finally { + emitter.close(); + } + } + + public static void main(String[] args) { + int exitCode = new CommandLine(new SchemaTron()).execute(args); + System.exit(exitCode); + } +} diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/java/io/datahubproject/schematron/SchemaTranslatorTest.java b/metadata-integration/java/datahub-schematron/cli/src/test/java/io/datahubproject/schematron/SchemaTranslatorTest.java new file mode 100644 index 00000000000000..bb11beb00729e7 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/src/test/java/io/datahubproject/schematron/SchemaTranslatorTest.java @@ -0,0 +1,149 @@ +package io.datahubproject.schematron; + +import static org.testng.Assert.assertEquals; + +import io.datahubproject.schematron.cli.SchemaTron; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import picocli.CommandLine; + +public class SchemaTranslatorTest { + private static final String TEST_RESOURCES_DIR = "src/test/resources"; + private static final String TEMP_OUTPUT_DIR = "build/test-outputs"; + private static final String PYTHON_SCRIPT = "scripts/avro_schema_to_mce.py"; + private static final String DIFF_SCRIPT = "scripts/mce_diff.py"; + private static final String VENV_PATH = + "../../../../metadata-ingestion/venv"; // Adjust this path to your venv location + + @BeforeClass + public static void setup() { + // Create output directory if it doesn't exist + new File(TEMP_OUTPUT_DIR).mkdirs(); + + // Verify venv exists + if (!new File(VENV_PATH).exists()) { + throw new RuntimeException("Virtual environment not found at " + VENV_PATH); + } + } + + @DataProvider(name = "schemaFiles") + public Object[][] getSchemaFiles() throws Exception { + List schemaFiles = + Files.walk(Paths.get(TEST_RESOURCES_DIR)) + .filter(path -> path.toString().endsWith(".avsc")) + .collect(Collectors.toList()); + + Object[][] testData = new Object[schemaFiles.size()][1]; + for (int i = 0; i < schemaFiles.size(); i++) { + testData[i][0] = schemaFiles.get(i); + } + return testData; + } + + @Test(dataProvider = "schemaFiles") + public void testSchemaTranslations(Path schemaFile) throws Exception { + compareTranslations(schemaFile); + } + + private ProcessBuilder createPythonProcessBuilder(String... args) { + ProcessBuilder pb; + String os = System.getProperty("os.name").toLowerCase(); + + if (os.contains("windows")) { + // Windows paths + String pythonPath = Paths.get(VENV_PATH, "Scripts", "python").toString(); + pb = + new ProcessBuilder( + Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new)); + } else { + // Unix-like paths + String pythonPath = Paths.get(VENV_PATH, "bin", "python").toString(); + pb = + new ProcessBuilder( + Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new)); + } + + // Add virtual environment to PYTHONPATH + Map env = pb.environment(); + String sitePkgPath = + Paths.get( + VENV_PATH, + os.contains("windows") ? "Lib/site-packages" : "lib/python3.x/site-packages") + .toString(); + + String pythonPath = env.getOrDefault("PYTHONPATH", ""); + env.put("PYTHONPATH", pythonPath + File.pathSeparator + sitePkgPath); + + return pb.inheritIO(); + } + + private void compareTranslations(Path schemaFile) throws Exception { + String baseName = schemaFile.getFileName().toString().replace(".avsc", ""); + String javaOutput = TEMP_OUTPUT_DIR + "/" + baseName + "_java.json"; + String pythonOutput = TEMP_OUTPUT_DIR + "/" + baseName + "_python.json"; + String diffFile = schemaFile.getParent().toString() + "/diffs/" + baseName + "_diff.json"; + + // Test if diffFile exists + File diff = new File(diffFile); + if (!diff.exists()) { + diffFile = null; + } + + // Run Python translator + Process pythonProcess = + createPythonProcessBuilder( + PYTHON_SCRIPT, + "--platform", + "datahub", + "--input-file", + schemaFile.toString(), + "--output-file", + pythonOutput) + .inheritIO() + .start(); + + int pythonExitCode = pythonProcess.waitFor(); + assertEquals(pythonExitCode, 0, "Python translation failed"); + + // Run Java translator directly using SchemaTron + SchemaTron schemaTron = new SchemaTron(); + int javaExitCode = + new CommandLine(schemaTron) + .execute( + "-i", + schemaFile.toAbsolutePath().toString(), + "--sink", + "file", + "--output-file", + javaOutput, + "--platform", + "datahub"); + + assertEquals(javaExitCode, 0, "Java translation failed"); + + // Compare outputs + // if diffFile is not provided, we just compare the outputs + ProcessBuilder diffProcessBuilder; + if (diffFile == null) { + diffProcessBuilder = createPythonProcessBuilder(DIFF_SCRIPT, pythonOutput, javaOutput); + } else { + diffProcessBuilder = + createPythonProcessBuilder( + DIFF_SCRIPT, pythonOutput, javaOutput, "--golden-diff-file", diffFile); + } + + Process diffProcess = diffProcessBuilder.inheritIO().start(); + + int diffExitCode = diffProcess.waitFor(); + assertEquals(diffExitCode, 0, "Outputs differ for " + schemaFile.getFileName()); + } +} diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile.avsc b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile.avsc new file mode 100644 index 00000000000000..81f8b0e54b11e0 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile.avsc @@ -0,0 +1,456 @@ +{ + "type": "record", + "name": "CustomerProfile", + "namespace": "com.example.customer", + "doc": "A complex customer profile schema demonstrating various union types and optional fields", + "fields": [ + { + "name": "customerId", + "type": { + "type": "string", + "logicalType": "uuid" + }, + "doc": "Unique identifier for the customer" + }, + { + "name": "identificationDocument", + "type": [ + "null", + { + "type": "record", + "name": "Passport", + "fields": [ + { + "name": "passportNumber", + "type": "string" + }, + { + "name": "expiryDate", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "DriversLicense", + "fields": [ + { + "name": "licenseNumber", + "type": "string" + }, + { + "name": "state", + "type": "string" + }, + { + "name": "validUntil", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "NationalID", + "fields": [ + { + "name": "idNumber", + "type": "string" + }, + { + "name": "country", + "type": "string" + } + ] + } + ], + "default": null, + "doc": "Customer's identification document - can be passport, driver's license, or national ID" + }, + { + "name": "contactInfo", + "type": { + "type": "record", + "name": "ContactInformation", + "fields": [ + { + "name": "primaryContact", + "type": [ + { + "type": "record", + "name": "EmailContact", + "fields": [ + { + "name": "emailAddress", + "type": "string" + }, + { + "name": "isVerified", + "type": "boolean", + "default": false + } + ] + }, + { + "type": "record", + "name": "PhoneContact", + "fields": [ + { + "name": "countryCode", + "type": "string" + }, + { + "name": "number", + "type": "string" + }, + { + "name": "type", + "type": { + "type": "enum", + "name": "PhoneType", + "symbols": [ + "MOBILE", + "LANDLINE" + ] + } + } + ] + } + ], + "doc": "Primary contact method - either email or phone" + }, + { + "name": "alternativeContacts", + "type": { + "type": "array", + "items": [ + "null", + "EmailContact", + "PhoneContact" + ] + }, + "default": [], + "doc": "List of alternative contact methods" + } + ] + } + }, + { + "name": "addresses", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "Address", + "fields": [ + { + "name": "type", + "type": { + "type": "enum", + "name": "AddressType", + "symbols": [ + "RESIDENTIAL", + "BUSINESS", + "SHIPPING" + ] + }, + "default": "RESIDENTIAL" + }, + { + "name": "street", + "type": "string" + }, + { + "name": "city", + "type": "string" + }, + { + "name": "state", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "country", + "type": "string" + }, + { + "name": "postalCode", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "validationStatus", + "type": [ + "null", + { + "type": "record", + "name": "AddressValidation", + "fields": [ + { + "name": "isValid", + "type": "boolean" + }, + { + "name": "verificationDate", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + }, + { + "name": "verificationMethod", + "type": { + "type": "enum", + "name": "VerificationMethod", + "symbols": [ + "MANUAL", + "AUTOMATED" + ] + } + } + ] + } + ], + "default": null + } + ] + } + }, + "doc": "Customer's addresses with validation information" + }, + { + "name": "preferences", + "type": { + "type": "map", + "values": [ + "null", + "string", + "boolean", + { + "type": "record", + "name": "FrequencyPreference", + "fields": [ + { + "name": "frequency", + "type": { + "type": "enum", + "name": "Frequency", + "symbols": [ + "DAILY", + "WEEKLY", + "MONTHLY" + ] + } + }, + { + "name": "enabled", + "type": "boolean", + "default": true + }, + { + "name": "lastUpdated", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] + } + ] + }, + "doc": "Customer preferences with various possible value types" + }, + { + "name": "subscriptionHistory", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "Subscription", + "fields": [ + { + "name": "planName", + "type": "string" + }, + { + "name": "startDate", + "type": { + "type": "long", + "logicalType": "date" + } + }, + { + "name": "endDate", + "type": [ + "null", + { + "type": "long", + "logicalType": "date" + } + ], + "default": null + }, + { + "name": "status", + "type": { + "type": "enum", + "name": "SubscriptionStatus", + "symbols": [ + "ACTIVE", + "CANCELLED", + "EXPIRED", + "SUSPENDED" + ] + } + }, + { + "name": "paymentMethod", + "type": [ + "null", + { + "type": "record", + "name": "PaymentMethod", + "fields": [ + { + "name": "type", + "type": { + "type": "enum", + "name": "PaymentType", + "symbols": [ + "CREDIT_CARD", + "DEBIT_CARD", + "BANK_TRANSFER", + "DIGITAL_WALLET" + ] + } + }, + { + "name": "lastFourDigits", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "expiryDate", + "type": [ + "null", + { + "type": "long", + "logicalType": "date" + } + ], + "default": null + } + ] + } + ], + "default": null + } + ] + } + } + ], + "default": null, + "doc": "Historical record of customer subscriptions" + }, + { + "name": "metadata", + "type": { + "type": "map", + "values": [ + "null", + "string", + "long", + "boolean", + { + "type": "record", + "name": "MetadataValue", + "fields": [ + { + "name": "value", + "type": [ + "null", + "string", + "long", + "boolean" + ], + "default": null + }, + { + "name": "timestamp", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + }, + { + "name": "source", + "type": "string" + } + ] + } + ] + }, + "doc": "Flexible metadata storage with various possible value types" + }, + { + "name": "tags", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "Tag", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "value", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "score", + "type": [ + "null", + "double" + ], + "default": null + }, + { + "name": "addedAt", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] + } + } + ], + "default": null, + "doc": "Optional tags associated with the customer profile" + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile2.avsc b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile2.avsc new file mode 100644 index 00000000000000..b8c7654ea072a2 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/CustomerProfile2.avsc @@ -0,0 +1,244 @@ +{ + "type": "record", + "name": "CustomerProfile2", + "namespace": "com.example.customer", + "doc": "A complex customer profile schema demonstrating various union types and optional fields", + "fields": [ + { + "name": "customerId", + "type": { + "type": "string", + "logicalType": "uuid" + }, + "doc": "Unique identifier for the customer" + }, + { + "name": "identificationDocument", + "type": [ + "null", + { + "type": "record", + "name": "Passport", + "fields": [ + { + "name": "passportNumber", + "type": "string" + }, + { + "name": "expiryDate", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "DriversLicense", + "fields": [ + { + "name": "licenseNumber", + "type": "string" + }, + { + "name": "state", + "type": "string" + }, + { + "name": "validUntil", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "NationalID", + "fields": [ + { + "name": "idNumber", + "type": "string" + }, + { + "name": "country", + "type": "string" + } + ] + } + ], + "default": null, + "doc": "Customer's identification document" + }, + { + "name": "contactInfo", + "type": { + "type": "record", + "name": "ContactInformation", + "fields": [ + { + "name": "primaryEmailContact", + "type": [ + "null", + { + "type": "record", + "name": "PrimaryEmailContact", + "fields": [ + { + "name": "emailAddress", + "type": "string" + }, + { + "name": "isVerified", + "type": "boolean", + "default": false + } + ] + } + ], + "default": null + }, + { + "name": "primaryPhoneContact", + "type": [ + "null", + { + "type": "record", + "name": "PrimaryPhoneContact", + "fields": [ + { + "name": "countryCode", + "type": "string" + }, + { + "name": "number", + "type": "string" + }, + { + "name": "type", + "type": { + "type": "enum", + "name": "PhoneType", + "symbols": [ + "MOBILE", + "LANDLINE" + ] + } + } + ] + } + ], + "default": null + }, + { + "name": "alternativeEmailContacts", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "AlternativeEmailContact", + "fields": [ + { + "name": "emailAddress", + "type": "string" + }, + { + "name": "isVerified", + "type": "boolean", + "default": false + } + ] + } + }, + "default": [] + }, + { + "name": "alternativePhoneContacts", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "AlternativePhoneContact", + "fields": [ + { + "name": "countryCode", + "type": "string" + }, + { + "name": "number", + "type": "string" + }, + { + "name": "type", + "type": "PhoneType" + } + ] + } + }, + "default": [] + } + ] + } + }, + { + "name": "preferences", + "type": { + "type": "record", + "name": "Preferences", + "fields": [ + { + "name": "simplePreferences", + "type": { + "type": "map", + "values": [ + "null", + "string", + "boolean" + ] + }, + "default": {} + }, + { + "name": "frequencyPreferences", + "type": { + "type": "map", + "values": { + "type": "record", + "name": "FrequencyPreference", + "fields": [ + { + "name": "frequency", + "type": { + "type": "enum", + "name": "Frequency", + "symbols": [ + "DAILY", + "WEEKLY", + "MONTHLY" + ] + } + }, + { + "name": "enabled", + "type": "boolean", + "default": true + }, + { + "name": "lastUpdated", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] + } + }, + "default": {} + } + ] + } + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/FlatUser.avsc b/metadata-integration/java/datahub-schematron/cli/src/test/resources/FlatUser.avsc new file mode 100644 index 00000000000000..c796878c32ae41 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/FlatUser.avsc @@ -0,0 +1,45 @@ +{ + "type": "record", + "name": "FlatUser", + "namespace": "com.example", + "fields": [ + { + "name": "id", + "type": "int", + "doc": "The unique identifier for a user", + "default": -1, + "metadata": { + "key1": "value1", + "key2": "value2" + } + }, + { + "name": "username", + "type": "string", + "doc": "The username of the user" + }, + { + "name": "email", + "type": "string", + "doc": "The email of the user" + }, + { + "name": "age", + "type": "int", + "doc": "The age of the user" + }, + { + "name": "isActive", + "type": "boolean", + "doc": "Whether the user is active or not" + }, + { + "name": "registrationDate", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + }, + "doc": "The registration date of the user" + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile2_diff.json b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile2_diff.json new file mode 100644 index 00000000000000..d4677d722a0cb2 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile2_diff.json @@ -0,0 +1,125 @@ +{ + "urn:li:dataset:(urn:li:dataPlatform:datahub,com.example.customer.CustomerProfile2,PROD)": { + "schemaMetadata": { + "added": {}, + "modified": [ + "fields" + ], + "modified_details": { + "fields": { + "added": {}, + "modified": [ + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=PrimaryEmailContact].primaryEmailContact.[type=boolean].isVerified", + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts", + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts.[type=boolean].isVerified", + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativePhoneContact].alternativePhoneContacts", + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences", + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences.[type=boolean].enabled", + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=boolean].simplePreferences", + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=string].simplePreferences", + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].simplePreferences", + "[version=2.0].[type=CustomerProfile2].[type=union].[type=DriversLicense].identificationDocument", + "[version=2.0].[type=CustomerProfile2].[type=union].[type=NationalID].identificationDocument", + "[version=2.0].[type=CustomerProfile2].[type=union].[type=Passport].identificationDocument", + "[version=2.0].[type=CustomerProfile2].[type=union].identificationDocument" + ], + "modified_details": { + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=PrimaryEmailContact].primaryEmailContact.[type=boolean].isVerified": { + "description": { + "after": null, + "before": "\nField default value: False", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=PrimaryEmailContact].primaryEmailContact.[type=boolean].isVerified" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts": { + "description": { + "after": null, + "before": "\nField default value: []", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts.[type=boolean].isVerified": { + "description": { + "after": null, + "before": "\nField default value: False", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativeEmailContact].alternativeEmailContacts.[type=boolean].isVerified" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativePhoneContact].alternativePhoneContacts": { + "description": { + "after": null, + "before": "\nField default value: []", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=ContactInformation].contactInfo.[type=array].[type=AlternativePhoneContact].alternativePhoneContacts" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences": { + "description": { + "after": null, + "before": "\nField default value: {}", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences.[type=boolean].enabled": { + "description": { + "after": null, + "before": "\nField default value: True", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=FrequencyPreference].frequencyPreferences.[type=boolean].enabled" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=boolean].simplePreferences": { + "description": { + "after": null, + "before": "\nField default value: {}", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=boolean].simplePreferences" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=string].simplePreferences": { + "description": { + "after": null, + "before": "\nField default value: {}", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].[type=string].simplePreferences" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].simplePreferences": { + "description": { + "after": null, + "before": "\nField default value: {}", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=Preferences].preferences.[type=map].[type=union].simplePreferences" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=union].[type=DriversLicense].identificationDocument": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].[type=DriversLicense].identificationDocument" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=union].[type=NationalID].identificationDocument": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].[type=NationalID].identificationDocument" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=union].[type=Passport].identificationDocument": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].[type=Passport].identificationDocument" + } + }, + "[version=2.0].[type=CustomerProfile2].[type=union].identificationDocument": { + "description": { + "after": "Customer's identification document\nField default value: null", + "before": "Customer's identification document", + "identifier": "[version=2.0].[type=CustomerProfile2].[type=union].identificationDocument" + } + } + }, + "removed": {} + } + }, + "removed": {} + } + } +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile_diff.json b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile_diff.json new file mode 100644 index 00000000000000..4bf0e1074d9a4a --- /dev/null +++ b/metadata-integration/java/datahub-schematron/cli/src/test/resources/diffs/CustomerProfile_diff.json @@ -0,0 +1,181 @@ +{ + "urn:li:dataset:(urn:li:dataPlatform:datahub,com.example.customer.CustomerProfile,PROD)": { + "schemaMetadata": { + "added": {}, + "modified": [ + "fields" + ], + "modified_details": { + "fields": { + "added": { + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=boolean].isVerified": { + "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=boolean].isVerified", + "isPartOfKey": false, + "nativeDataType": "boolean", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + } + }, + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=string].emailAddress": { + "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts.[type=string].emailAddress", + "isPartOfKey": false, + "nativeDataType": "string", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + } + }, + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=enum].type": { + "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=enum].type", + "isPartOfKey": false, + "nativeDataType": "Enum", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.EnumType": {} + } + } + }, + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].countryCode": { + "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].countryCode", + "isPartOfKey": false, + "nativeDataType": "string", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + } + }, + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].number": { + "fieldPath": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts.[type=string].number", + "isPartOfKey": false, + "nativeDataType": "string", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + } + } + }, + "modified": [ + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts", + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts", + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].alternativeContacts", + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=union].[type=EmailContact].primaryContact.[type=boolean].isVerified", + "[version=2.0].[type=CustomerProfile].[type=array].[type=Address].addresses.[type=enum].type", + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=FrequencyPreference].preferences.[type=boolean].enabled", + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=boolean].value", + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=long].value", + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=string].value", + "[version=2.0].[type=CustomerProfile].[type=union].[type=DriversLicense].identificationDocument", + "[version=2.0].[type=CustomerProfile].[type=union].[type=NationalID].identificationDocument", + "[version=2.0].[type=CustomerProfile].[type=union].[type=Passport].identificationDocument", + "[version=2.0].[type=CustomerProfile].[type=union].identificationDocument" + ], + "modified_details": { + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts": { + "description": { + "after": "List of alternative contact methods", + "before": "List of alternative contact methods\nField default value: []", + "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=EmailContact].alternativeContacts" + } + }, + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts": { + "description": { + "after": "List of alternative contact methods", + "before": "List of alternative contact methods\nField default value: []", + "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].[type=PhoneContact].alternativeContacts" + } + }, + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].alternativeContacts": { + "description": { + "after": "List of alternative contact methods", + "before": "List of alternative contact methods\nField default value: []", + "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=array].[type=union].alternativeContacts" + } + }, + "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=union].[type=EmailContact].primaryContact.[type=boolean].isVerified": { + "description": { + "after": null, + "before": "\nField default value: False", + "identifier": "[version=2.0].[type=CustomerProfile].[type=ContactInformation].contactInfo.[type=union].[type=EmailContact].primaryContact.[type=boolean].isVerified" + } + }, + "[version=2.0].[type=CustomerProfile].[type=array].[type=Address].addresses.[type=enum].type": { + "description": { + "after": null, + "before": "\nField default value: RESIDENTIAL", + "identifier": "[version=2.0].[type=CustomerProfile].[type=array].[type=Address].addresses.[type=enum].type" + } + }, + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=FrequencyPreference].preferences.[type=boolean].enabled": { + "description": { + "after": null, + "before": "\nField default value: True", + "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=FrequencyPreference].preferences.[type=boolean].enabled" + } + }, + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=boolean].value": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=boolean].value" + } + }, + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=long].value": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=long].value" + } + }, + "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=string].value": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile].[type=map].[type=union].[type=MetadataValue].metadata.[type=union].[type=string].value" + } + }, + "[version=2.0].[type=CustomerProfile].[type=union].[type=DriversLicense].identificationDocument": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile].[type=union].[type=DriversLicense].identificationDocument" + } + }, + "[version=2.0].[type=CustomerProfile].[type=union].[type=NationalID].identificationDocument": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile].[type=union].[type=NationalID].identificationDocument" + } + }, + "[version=2.0].[type=CustomerProfile].[type=union].[type=Passport].identificationDocument": { + "nullable": { + "after": false, + "before": true, + "identifier": "[version=2.0].[type=CustomerProfile].[type=union].[type=Passport].identificationDocument" + } + }, + "[version=2.0].[type=CustomerProfile].[type=union].identificationDocument": { + "description": { + "after": "Customer's identification document - can be passport, driver's license, or national ID\nField default value: null", + "before": "Customer's identification document - can be passport, driver's license, or national ID", + "identifier": "[version=2.0].[type=CustomerProfile].[type=union].identificationDocument" + } + } + }, + "removed": {} + } + }, + "removed": {} + } + } +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/lib/build.gradle b/metadata-integration/java/datahub-schematron/lib/build.gradle new file mode 100644 index 00000000000000..83dec1039f7be0 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/build.gradle @@ -0,0 +1,130 @@ +plugins { + id("com.palantir.git-version") apply false +} +apply plugin: 'java' +apply plugin: 'jacoco' +apply plugin: 'signing' +apply plugin: 'io.codearte.nexus-staging' +apply plugin: 'maven-publish' +apply from: '../../versioning.gradle' + +dependencies { + + implementation project(':entity-registry') +// +// // Jackson dependencies - use the same versions as in the parent project +// implementation 'com.fasterxml.jackson.core:jackson-core:2.12.3' +// implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3' +// implementation 'com.fasterxml.jackson.core:jackson-annotations:2.12.3' + + // Core dependencies +// implementation externalDependency.guava +// implementation externalDependency.gson +// implementation externalDependency.commonsCli +// implementation externalDependency.slf4jApi +// implementation externalDependency.jacksonCore + + // Schema format dependencies +// implementation externalDependency.protobuf + implementation externalDependency.avro +// implementation 'org.apache.thrift:libthrift:0.16.0' +// implementation 'io.swagger.parser.v3:swagger-parser:2.1.12' + + // Utilities + compileOnly externalDependency.lombok + annotationProcessor externalDependency.lombok + + // Testing + testImplementation externalDependency.testng + testImplementation 'org.mockito:mockito-core:5.3.1' +} + +jacocoTestReport { + dependsOn test +} + +test.finalizedBy jacocoTestReport + +task checkShadowJar(type: Exec) { + commandLine 'sh', '-c', 'scripts/check_jar.sh' +} + +configurations { + provided + implementation.extendsFrom provided +} + +java { + withJavadocJar() + withSourcesJar() +} + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + + pom { + name = 'Datahub Schematron' + groupId = 'io.acryl' + artifactId = 'datahub-schematron' + description = 'DataHub schema translation library for converting between different schema formats using DataHub as an intermediate representation' + url = 'https://datahubproject.io' + + scm { + connection = 'scm:git:git://github.com/datahub-project/datahub.git' + developerConnection = 'scm:git:ssh://github.com:datahub-project/datahub.git' + url = 'https://github.com/datahub-project/datahub.git' + } + + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + + developers { + developer { + id = 'datahub' + name = 'Datahub' + email = 'datahub@acryl.io' + } + } + } + } + } + + repositories { + maven { + def releasesRepoUrl = "https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/" + def snapshotsRepoUrl = "https://s01.oss.sonatype.org/content/repositories/snapshots/" + def ossrhUsername = System.getenv('RELEASE_USERNAME') + def ossrhPassword = System.getenv('RELEASE_PASSWORD') + credentials { + username ossrhUsername + password ossrhPassword + } + url = version.endsWith('SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl + } + } +} + +signing { + def signingKey = findProperty("signingKey") + def signingPassword = System.getenv("SIGNING_PASSWORD") + // Only require signing if we have the signing key property + required = signingKey != null + + if (signingKey != null) { + useInMemoryPgpKeys(signingKey, signingPassword) + sign publishing.publications.mavenJava + } + +} + +nexusStaging { + serverUrl = "https://s01.oss.sonatype.org/service/local/" + username = System.getenv("NEXUS_USERNAME") + password = System.getenv("NEXUS_PASSWORD") +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/SchemaConverter.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/SchemaConverter.java new file mode 100644 index 00000000000000..cb364f2c7a1a2d --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/SchemaConverter.java @@ -0,0 +1,25 @@ +package io.datahubproject.schematron.converters; + +import com.linkedin.common.urn.DataPlatformUrn; +import com.linkedin.schema.SchemaMetadata; + +/** Base interface for converting between different schema formats. */ +public interface SchemaConverter { + /** + * Converts a schema into DataHub's SchemaField format. + * + * @param schema The source schema to convert + * @param isKeySchema Whether this represents a key schema + * @param defaultNullable Default nullable setting for fields + * @param platformUrn Data platform urn + * @param rawSchemaString Raw schema string (if available). When provided - it will be used to + * generate the schema fingerprint + * @return List of SchemaFields representing the schema in DataHub's format + */ + SchemaMetadata toDataHubSchema( + T schema, + boolean isKeySchema, + boolean defaultNullable, + DataPlatformUrn platformUrn, + String rawSchemaString); +} diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java new file mode 100644 index 00000000000000..c199f8e6dcb92e --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java @@ -0,0 +1,607 @@ +package io.datahubproject.schematron.converters.avro; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.common.urn.DataPlatformUrn; +import com.linkedin.schema.*; +import io.datahubproject.schematron.converters.SchemaConverter; +import io.datahubproject.schematron.models.*; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.Supplier; +import lombok.Builder; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.JsonProperties; +import org.apache.avro.LogicalType; +import org.apache.avro.Schema; +import org.apache.avro.SchemaNormalization; + +/** Converts Avro schemas to DataHub's schema format following SchemaFieldPath Specification V2. */ +@Slf4j +@Builder +public class AvroSchemaConverter implements SchemaConverter { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Map> LOGICAL_TYPE_MAPPING; + + static { + Map> logicalTypeMap = new HashMap<>(); + logicalTypeMap.put("date", () -> SchemaFieldDataType.Type.create(new DateType())); + logicalTypeMap.put("time-micros", () -> SchemaFieldDataType.Type.create(new TimeType())); + logicalTypeMap.put("time-millis", () -> SchemaFieldDataType.Type.create(new TimeType())); + logicalTypeMap.put("timestamp-micros", () -> SchemaFieldDataType.Type.create(new TimeType())); + logicalTypeMap.put("timestamp-millis", () -> SchemaFieldDataType.Type.create(new TimeType())); + logicalTypeMap.put("decimal", () -> SchemaFieldDataType.Type.create(new NumberType())); + logicalTypeMap.put("uuid", () -> SchemaFieldDataType.Type.create(new StringType())); + LOGICAL_TYPE_MAPPING = Collections.unmodifiableMap(logicalTypeMap); + } + + private SchemaFieldDataType.Type getTypeFromLogicalType(Schema schema) { + LogicalType logicalType = schema.getLogicalType(); + if (logicalType != null) { + Supplier typeSupplier = + LOGICAL_TYPE_MAPPING.get(logicalType.getName()); + if (typeSupplier != null) { + return typeSupplier.get(); + } + } + return getBaseType(schema); + } + + private SchemaFieldDataType.Type getBaseType(Schema schema) { + switch (schema.getType()) { + case BOOLEAN: + return SchemaFieldDataType.Type.create(new BooleanType()); + case INT: + case LONG: + case FLOAT: + case DOUBLE: + return SchemaFieldDataType.Type.create(new NumberType()); + case STRING: + return SchemaFieldDataType.Type.create(new StringType()); + case BYTES: + return SchemaFieldDataType.Type.create(new BytesType()); + case FIXED: + return SchemaFieldDataType.Type.create(new FixedType()); + case ENUM: + return SchemaFieldDataType.Type.create(new EnumType()); + case ARRAY: + return SchemaFieldDataType.Type.create(new ArrayType()); + case MAP: + return SchemaFieldDataType.Type.create(new MapType()); + case RECORD: + return SchemaFieldDataType.Type.create(new RecordType()); + case UNION: + return SchemaFieldDataType.Type.create(new UnionType()); + default: + return SchemaFieldDataType.Type.create(new NullType()); + } + } + + private String getFieldType(Schema schema) { + // For the field path, we just want the base type without the logical type + return schema.getType().getName().toLowerCase(); + } + + private String getNativeDataType(Schema schema) { + // For native data type, we can include the logical type information + LogicalType logicalType = schema.getLogicalType(); + if (logicalType != null) { + return schema.getType().getName().toLowerCase() + "(" + logicalType.getName() + ")"; + } + return schema.getType().getName().toLowerCase(); + } + + @Override + public SchemaMetadata toDataHubSchema( + Schema schema, + boolean isKeySchema, + boolean defaultNullable, + DataPlatformUrn platformUrn, + String rawSchemaString) { + + try { + byte[] fingerprintBytes = null; + try { + if (rawSchemaString != null) { + String canonicalForm = SchemaNormalization.toParsingForm(schema); + log.debug("Length of canonical form: {}", canonicalForm.length()); + log.debug("Canonical form: {}", canonicalForm); + fingerprintBytes = + SchemaNormalization.fingerprint( + "MD5", rawSchemaString.getBytes(StandardCharsets.UTF_8)); + } else { + fingerprintBytes = SchemaNormalization.parsingFingerprint("MD5", schema); + } + } catch (Exception e) { + log.error("Failed to compute schema fingerprint", e); + } + + String schemaHash = ""; + if (fingerprintBytes != null) { + // Convert to hex string + StringBuilder hexString = new StringBuilder(); + for (byte b : fingerprintBytes) { + hexString.append(String.format("%02x", b)); + } + schemaHash = hexString.toString(); + } + + List fields = new ArrayList<>(); + FieldPath basePath = new FieldPath(); + basePath.setKeySchema(isKeySchema); + + // Add the record type to the base path + if (schema.getType() == Schema.Type.RECORD) { + basePath = basePath.expandType(schema.getName(), schema.toString()); + } + + processSchema(schema, basePath, defaultNullable, fields); + + return new SchemaMetadata() + .setSchemaName(schema.getName()) + .setPlatform(platformUrn) + .setVersion(0) + .setHash(schemaHash) + .setPlatformSchema( + SchemaMetadata.PlatformSchema.create( + new OtherSchema().setRawSchema(schema.toString()))) + .setFields(new SchemaFieldArray(fields)); + + } catch (Exception e) { + log.error("Failed to convert Avro schema", e); + throw new RuntimeException("Failed to convert Avro schema", e); + } + } + + private void processSchema( + Schema schema, FieldPath fieldPath, boolean defaultNullable, List fields) { + if (schema.getType() == Schema.Type.RECORD) { + for (Schema.Field field : schema.getFields()) { + processField(field, fieldPath, defaultNullable, fields); + } + } + } + + private void processField( + Schema.Field field, FieldPath fieldPath, boolean defaultNullable, List fields) { + processField(field, fieldPath, defaultNullable, fields, false, null); + } + + private void processField( + Schema.Field field, + FieldPath fieldPath, + boolean defaultNullable, + List fields, + boolean nullableOverride) { + processField(field, fieldPath, defaultNullable, fields, nullableOverride, null); + } + + private void processField( + Schema.Field field, + FieldPath fieldPath, + boolean defaultNullable, + List fields, + boolean nullableOverride, + DataHubType typeOverride) { + log.debug( + "Processing field: {}, Field path : {}, Field schema: {}", + field.name(), + fieldPath.asString(), + field.schema()); + Schema fieldSchema = field.schema(); + boolean isNullable = isNullable(fieldSchema, defaultNullable); + if (nullableOverride) { + // If a nullable override is provided, use the override value + isNullable = true; + } + if (typeOverride != null) { + // If a type override is provided, use the nullable value from the override + isNullable = nullableOverride; + } + log.debug( + "DefaultNullability: {}, Determined nullability for field name: {} at path: {} is {}", + defaultNullable, + field.name(), + fieldPath.asString(), + isNullable); + String discriminatedType = getDiscriminatedType(fieldSchema); + + FieldElement element = + new FieldElement(new ArrayList<>(), new ArrayList<>(), field.name(), typeOverride); + + FieldPath newPath = fieldPath.clonePlus(element); + + switch (fieldSchema.getType()) { + case RECORD: + processRecordField( + field, newPath, discriminatedType, defaultNullable, fields, isNullable, typeOverride); + break; + case ARRAY: + processArrayField(field, newPath, discriminatedType, defaultNullable, fields, isNullable); + break; + case MAP: + processMapField(field, newPath, discriminatedType, defaultNullable, fields, isNullable); + break; + case UNION: + processUnionField( + field, newPath, discriminatedType, defaultNullable, fields, isNullable, typeOverride); + break; + case ENUM: + processEnumField(field, newPath, discriminatedType, defaultNullable, fields, isNullable); + break; + default: + processPrimitiveField( + field, newPath, discriminatedType, defaultNullable, fields, isNullable); + break; + } + } + + private void processRecordField( + Schema.Field field, + FieldPath fieldPath, + String discriminatedType, + boolean defaultNullable, + List fields, + boolean isNullable, + DataHubType typeOverride) { + + log.debug("Record Field Path before expand: {}", fieldPath.asString()); + FieldPath recordPath = fieldPath.expandType(discriminatedType, field.schema().toString()); + log.debug("Record Field Path after expand: {}", recordPath.asString()); + + SchemaFieldDataType dataType = + typeOverride != null + ? typeOverride.asSchemaFieldType() + : new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())); + + // Add the record field itself + SchemaField recordField = + new SchemaField() + .setFieldPath(recordPath.asString()) + .setType(dataType) + .setNativeDataType(discriminatedType) + .setNullable(isNullable || defaultNullable) + .setIsPartOfKey(fieldPath.isKeySchema()); + + populateCommonProperties(field, recordField); + + fields.add(recordField); + + // Process nested fields + for (Schema.Field nestedField : field.schema().getFields()) { + processField(nestedField, recordPath, defaultNullable, fields); + } + } + + @SneakyThrows + private static void populateCommonProperties(Schema.Field field, SchemaField datahubField) { + // Create a new mutable HashMap to store combined properties + Map combinedProps = new HashMap<>(); + + // Add properties from field if any exist + Map fieldProps = field.getObjectProps(); + if (fieldProps != null) { + combinedProps.putAll(fieldProps); + } + + // Add properties from schema if any exist + Map schemaProps = field.schema().getObjectProps(); + if (schemaProps != null) { + combinedProps.putAll(schemaProps); + } + + // Only proceed with serialization if we have properties + if (!combinedProps.isEmpty()) { + try { + String jsonSerializedProps = OBJECT_MAPPER.writeValueAsString(combinedProps); + datahubField.setJsonProps(jsonSerializedProps); + } catch (Exception e) { + log.error( + "Non-fatal error. Failed to serialize schema properties for field: " + field.name(), e); + } + } + + // Set the description if it exists + if (field.doc() != null && !field.doc().isEmpty()) { + datahubField.setDescription(field.doc()); + if (field.hasDefaultValue()) { + Object defaultValue = field.defaultVal(); + // if the default value is the JSON NULL node, then we handle it differently + if (defaultValue == JsonProperties.NULL_VALUE) { + datahubField.setDescription( + datahubField.getDescription() + "\nField default value: null"); + } else { + datahubField.setDescription( + datahubField.getDescription() + + "\nField default value: " + + OBJECT_MAPPER.writeValueAsString(defaultValue)); + } + } + } + } + + private void processArrayField( + Schema.Field field, + FieldPath fieldPath, + String discriminatedType, + boolean defaultNullable, + List fields, + boolean isNullable) { + + Schema arraySchema = field.schema(); + Schema elementSchema = arraySchema.getElementType(); + String elementType = getDiscriminatedType(elementSchema); + + fieldPath = fieldPath.expandType("array", arraySchema); + // Set parent type for proper array handling + DataHubType arrayDataHubType = new DataHubType(ArrayType.class, elementType); + + // Process element type if it's complex + if (elementSchema.getType() == Schema.Type.RECORD + || elementSchema.getType() == Schema.Type.ARRAY + || elementSchema.getType() == Schema.Type.MAP + || elementSchema.getType() == Schema.Type.UNION) { + log.debug("Array Field Path before expand: {}", fieldPath.asString()); + fieldPath = fieldPath.popLast(); + fieldPath = + fieldPath.clonePlus(new FieldElement(List.of("array"), new ArrayList<>(), null, null)); + Schema.Field elementField = + new Schema.Field( + field.name(), + elementSchema, + elementSchema.getDoc() != null ? elementSchema.getDoc() : field.doc(), + null // TODO: What is the default value for an array element? + ); + processField(elementField, fieldPath, defaultNullable, fields, isNullable, arrayDataHubType); + } else { + + SchemaField arrayField = + new SchemaField() + .setFieldPath(fieldPath.asString()) + .setType(arrayDataHubType.asSchemaFieldType()) + .setNativeDataType("array(" + elementType + ")") + .setNullable(isNullable || defaultNullable) + .setIsPartOfKey(fieldPath.isKeySchema()); + + populateCommonProperties(field, arrayField); + log.debug("Array field path: {} with doc: {}", fieldPath.asString(), field.doc()); + fields.add(arrayField); + } + } + + private void processMapField( + Schema.Field field, + FieldPath fieldPath, + String discriminatedType, + boolean defaultNullable, + List fields, + boolean isNullable) { + + Schema mapSchema = field.schema(); + Schema valueSchema = mapSchema.getValueType(); + String valueType = getDiscriminatedType(valueSchema); + + DataHubType mapDataHubType = new DataHubType(MapType.class, valueType); + fieldPath = fieldPath.expandType("map", mapSchema); + + // Process value type if it's complex + if (valueSchema.getType() == Schema.Type.RECORD + || valueSchema.getType() == Schema.Type.ARRAY + || valueSchema.getType() == Schema.Type.MAP + || valueSchema.getType() == Schema.Type.UNION) { + Schema.Field valueField = + new Schema.Field( + field.name(), + valueSchema, + valueSchema.getDoc() != null ? valueSchema.getDoc() : field.doc(), + null // TODO: What is the default value for a map value? + ); // Nullability for map values follows the nullability of the map itself + FieldPath valueFieldPath = + fieldPath + .popLast() + .clonePlus(new FieldElement(List.of("map"), new ArrayList<>(), null, null)); + processField(valueField, valueFieldPath, defaultNullable, fields, isNullable, mapDataHubType); + } else { + SchemaField mapField = + new SchemaField() + .setFieldPath(fieldPath.asString()) + .setType(mapDataHubType.asSchemaFieldType()) + .setNativeDataType("map") + .setNullable(isNullable || defaultNullable) + .setIsPartOfKey(fieldPath.isKeySchema()); + + populateCommonProperties(field, mapField); + fields.add(mapField); + } + } + + private void processUnionField( + Schema.Field field, + FieldPath fieldPath, + String discriminatedType, + boolean defaultNullable, + List fields, + boolean isNullable, + DataHubType typeOverride) { + + List unionTypes = field.schema().getTypes(); + + // If this is just a nullable type (union with null), process the non-null type + // directly + if (unionTypes.size() == 2 && isNullable) { + Schema nonNullSchema = + unionTypes.stream() + .filter(s -> s.getType() != Schema.Type.NULL) + .findFirst() + .orElseThrow(); + + processField( + new Schema.Field(field.name(), nonNullSchema, field.doc()), + fieldPath.popLast(), + defaultNullable, + fields, + true); + return; + } + + log.debug("Union Field Path before expand: {}", fieldPath.asString()); + + // Otherwise, process as a true union type + DataHubType unionDataHubType = new DataHubType(UnionType.class, discriminatedType); + FieldPath unionFieldPath = fieldPath.expandType("union", field.schema().toString()); + log.debug("Union Field Path after expand: {}", unionFieldPath.asString()); + + SchemaField unionField = + new SchemaField() + .setFieldPath(unionFieldPath.asString()) + .setType( + typeOverride == null + ? unionDataHubType.asSchemaFieldType() + : typeOverride.asSchemaFieldType()) + .setNativeDataType("union") + .setNullable(isNullable || defaultNullable) + .setIsPartOfKey(fieldPath.isKeySchema()); + + populateCommonProperties(field, unionField); + fields.add(unionField); + + String unionDescription = field.doc() != null ? field.doc() : field.schema().getDoc(); + + // Process each union type + int typeIndex = 0; + for (Schema unionSchema : unionTypes) { + if (unionSchema.getType() != Schema.Type.NULL) { + log.debug("TypeIndex: {}, Field path : {}", typeIndex, fieldPath.asString()); + FieldPath indexedFieldPath = fieldPath.popLast(); + indexedFieldPath = + indexedFieldPath.clonePlus( + new FieldElement(List.of("union"), new ArrayList<>(), null, null)); + log.debug("TypeIndex: {}, Indexed Field path : {}", typeIndex, indexedFieldPath.asString()); + // FieldPath unionFieldPath = + // fieldPath.expandType(getDiscriminatedType(unionSchema), + // unionSchema.toString()); + log.debug("TypeIndex: {}, Union Field path : {}", typeIndex, unionFieldPath.asString()); + String unionFieldName = field.name(); + Schema.Field unionFieldInner = + new Schema.Field( + unionFieldName, + unionSchema, + unionSchema.getDoc() != null ? unionSchema.getDoc() : unionDescription, + null); + log.debug( + "TypeIndex: {}, Union Field path : {}, Doc: {}", + typeIndex, + unionFieldPath.asString(), + unionFieldInner.doc()); + processField(unionFieldInner, indexedFieldPath, defaultNullable, fields); + } + typeIndex++; + } + } + + private void processEnumField( + Schema.Field field, + FieldPath fieldPath, + String discriminatedType, + boolean defaultNullable, + List fields, + boolean isNullable) { + + fieldPath = fieldPath.expandType("enum", field.schema().toString()); + + String enumDescription = field.doc() != null ? field.doc() : ""; + enumDescription += + " Allowed symbols are: " + String.join(", ", field.schema().getEnumSymbols()); + + SchemaField enumField = + new SchemaField() + .setFieldPath(fieldPath.asString()) + .setType( + new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new EnumType()))) + .setNativeDataType("Enum") + .setNullable(isNullable || defaultNullable) + .setIsPartOfKey(fieldPath.isKeySchema()); + + populateCommonProperties(field, enumField); + + if (field.doc() != null && !field.doc().isEmpty()) { + enumField.setDescription(enumDescription); + } + + fields.add(enumField); + } + + @SneakyThrows + private void processPrimitiveField( + Schema.Field field, + FieldPath fieldPath, + String discriminatedType, + boolean defaultNullable, + List fields, + boolean isNullable) { + + fieldPath = fieldPath.expandType(discriminatedType, field.schema().toString()); + SchemaField primitiveField = + new SchemaField() + .setFieldPath(fieldPath.asString()) + .setType(new SchemaFieldDataType().setType(getTypeFromLogicalType(field.schema()))) + .setNativeDataType(getNativeDataType(field.schema())) + .setNullable(isNullable || defaultNullable) + .setIsPartOfKey(fieldPath.isKeySchema()); + + populateCommonProperties(field, primitiveField); + + fields.add(primitiveField); + } + + private boolean isNullable(Schema schema, boolean defaultNullable) { + if (schema.getType() == Schema.Type.UNION) { + return schema.getTypes().stream().anyMatch(type -> type.getType() == Schema.Type.NULL); + } + return defaultNullable; + } + + /** + * for record type we want to include the fully qualified name stripped of the namespace + * + * @param schema + * @return + */ + private String getDiscriminatedType(Schema schema) { + + if (schema.getType() == Schema.Type.RECORD) { + if (schema.getNamespace() != null) { + return schema.getFullName().substring(schema.getNamespace().length() + 1); + } else { + return schema.getFullName(); + } + } + return schema.getType().getName().toLowerCase(); + } + + private SchemaFieldDataType getPrimitiveFieldType(Schema schema) { + + SchemaFieldDataType fieldType = new SchemaFieldDataType(); + switch (schema.getType()) { + case BOOLEAN: + fieldType.setType(SchemaFieldDataType.Type.create(new BooleanType())); + break; + case INT: + case LONG: + case FLOAT: + case DOUBLE: + fieldType.setType(SchemaFieldDataType.Type.create(new NumberType())); + break; + case STRING: + fieldType.setType(SchemaFieldDataType.Type.create(new StringType())); + break; + case BYTES: + fieldType.setType(SchemaFieldDataType.Type.create(new BytesType())); + break; + default: + fieldType.setType(SchemaFieldDataType.Type.create(new NullType())); + } + return fieldType; + } +} diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/DataHubType.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/DataHubType.java new file mode 100644 index 00000000000000..ec6e8ce5a35547 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/DataHubType.java @@ -0,0 +1,40 @@ +package io.datahubproject.schematron.models; + +import com.linkedin.data.template.StringArray; +import com.linkedin.schema.*; +import lombok.Data; + +@Data +public class DataHubType { + private Class type; + private String nestedType; + + public DataHubType(Class type, String nestedType) { + this.type = type; + this.nestedType = nestedType; + } + + public SchemaFieldDataType asSchemaFieldType() { + if (type == UnionType.class) { + return new SchemaFieldDataType() + .setType( + SchemaFieldDataType.Type.create( + new UnionType() + .setNestedTypes(nestedType != null ? new StringArray(nestedType) : null))); + } else if (type == ArrayType.class) { + return new SchemaFieldDataType() + .setType( + SchemaFieldDataType.Type.create( + new ArrayType() + .setNestedType(nestedType != null ? new StringArray(nestedType) : null))); + } else if (type == MapType.class) { + return new SchemaFieldDataType() + .setType( + SchemaFieldDataType.Type.create( + new MapType() + .setKeyType("string") + .setValueType(nestedType != null ? nestedType : null))); + } + throw new IllegalArgumentException("Unexpected type " + type); + } +} diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldElement.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldElement.java new file mode 100644 index 00000000000000..6cdde845d95614 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldElement.java @@ -0,0 +1,38 @@ +package io.datahubproject.schematron.models; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import lombok.Data; + +@Data +public class FieldElement { + private List type; + private List schemaTypes; + private String name; + private DataHubType parentType; + + public FieldElement( + List type, List schemaTypes, String name, DataHubType parentType) { + this.type = type; + this.schemaTypes = schemaTypes; + this.name = name; + this.parentType = parentType; + } + + public FieldElement clone() { + return new FieldElement(new ArrayList<>(type), new ArrayList<>(schemaTypes), name, parentType); + } + + public String asString(boolean v2Format) { + if (v2Format) { + String typePrefix = + type.stream() + .map(innerType -> "[type=" + innerType + "]") + .collect(Collectors.joining(".")); + return name != null ? typePrefix + "." + name : typePrefix; + } else { + return name != null ? name : ""; + } + } +} diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java new file mode 100644 index 00000000000000..e51aa1221c54e0 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java @@ -0,0 +1,173 @@ +package io.datahubproject.schematron.models; + +import com.linkedin.schema.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import lombok.Data; +import lombok.NonNull; + +@Data +public class FieldPath { + public static final String EMPTY_FIELD_NAME = " "; + @NonNull private List path; + private boolean isKeySchema; + private boolean useV2PathsAlways; + + public FieldPath() { + this.path = new ArrayList<>(); + this.isKeySchema = false; + this.useV2PathsAlways = true; + } + + public void setPath(List path) { + if (path == null) { + throw new IllegalArgumentException("Path cannot be null"); + } + // Ensure that no element in the path is null + if (path.stream().anyMatch(Objects::isNull)) { + throw new IllegalArgumentException("Path cannot contain null elements"); + } + this.path = path; + } + + private boolean needsV2Path() { + if (useV2PathsAlways) { + return true; + } + if (isKeySchema) { + return true; + } + return path.stream() + .flatMap(element -> element.getType().stream()) + .anyMatch(t -> t.equals("union") || t.equals("array")); + } + + private void setParentTypeIfNotExists(DataHubType parentType) { + if (!path.isEmpty() && path.get(path.size() - 1).getParentType() == null) { + path.get(path.size() - 1).setParentType(parentType); + } + } + + private SchemaFieldDataType getTypeOverride() { + if (!path.isEmpty() && path.get(path.size() - 1).getParentType() != null) { + return path.get(path.size() - 1).getParentType().asSchemaFieldType(); + } + return null; + } + + private String getNativeTypeOverride() { + SchemaFieldDataType typeOverride = getTypeOverride(); + if (typeOverride != null) { + if (typeOverride.getType().isArrayType()) { + ArrayType arrayType = typeOverride.getType().getArrayType(); + return String.format( + "array(%s)", + arrayType.getNestedType() != null ? String.join(",", arrayType.getNestedType()) : ""); + } else if (typeOverride.getType().isMapType()) { + MapType mapType = typeOverride.getType().getMapType(); + return String.format("map(str,%s)", mapType.getValueType()); + } + } + return null; + } + + public String getRecursive(Map schema) { + String schemaStr = schema.toString(); + for (FieldElement p : path) { + for (int i = 0; i < p.getSchemaTypes().size(); i++) { + if (p.getSchemaTypes().get(i).equals(schemaStr)) { + return p.getType().get(i); + } + } + } + return null; + } + + public FieldPath popLast() { + FieldPath fpath = new FieldPath(); + fpath.setKeySchema(isKeySchema); + fpath.setPath(new ArrayList<>(path)); + fpath.getPath().remove(fpath.getPath().size() - 1); + return fpath; + } + + public FieldPath clonePlus(FieldElement element) { + FieldPath fpath = new FieldPath(); + fpath.setKeySchema(isKeySchema); + fpath.setPath(new ArrayList<>(path)); + fpath.getPath().add(element); + return fpath; + } + + // TODO: Why is typeSchema an Object? + public FieldPath expandType(String type, Object typeSchema) { + FieldPath fpath = new FieldPath(); + fpath.setKeySchema(isKeySchema); + fpath.setPath(path.stream().map(FieldElement::clone).collect(Collectors.toList())); + + if (!fpath.getPath().isEmpty()) { + FieldElement lastElement = fpath.getPath().get(fpath.getPath().size() - 1); + lastElement.getType().add(type); + lastElement.getSchemaTypes().add(typeSchema.toString()); + } else { + fpath + .getPath() + .add( + new FieldElement( + new ArrayList<>(List.of(type)), + new ArrayList<>(List.of(typeSchema.toString())), + null, + null)); + } + return fpath; + } + + public boolean hasFieldName() { + return path.stream().anyMatch(f -> f.getName() != null); + } + + public boolean ensureFieldName() { + if (!hasFieldName()) { + if (path.isEmpty()) { + path.add(new FieldElement(new ArrayList<>(), new ArrayList<>(), null, null)); + } + path.get(path.size() - 1).setName(EMPTY_FIELD_NAME); + } + return true; + } + + public String asString() { + boolean v2Format = needsV2Path(); + List prefix = new ArrayList<>(); + + if (v2Format) { + prefix.add("[version=2.0]"); + if (isKeySchema) { + prefix.add("[key=True]"); + } + } + + if (!path.isEmpty()) { + return String.join(".", prefix) + + "." + + path.stream().map(f -> f.asString(v2Format)).collect(Collectors.joining(".")); + } else { + return String.join(".", prefix); + } + } + + public String dump() { + StringBuilder sb = new StringBuilder(); + sb.append("FieldPath: "); + sb.append(this.asString()); + for (FieldElement f : path) { + sb.append(f.getName()); + sb.append(" "); + sb.append(f.getSchemaTypes().toString()); + } + return sb.toString(); + } +} diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/utils/Constants.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/utils/Constants.java new file mode 100644 index 00000000000000..b41d2d88c9dc0e --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/utils/Constants.java @@ -0,0 +1,12 @@ +package io.datahubproject.schematron.utils; + +/** Constants used throughout the schema conversion process. */ +public final class Constants { + private Constants() {} + + public static final String ADD_TAG_OPERATION = "ADD_TAG"; + public static final String ADD_TERM_OPERATION = "ADD_TERM"; + + public static final String TAG_URN_PREFIX = "urn:li:tag:"; + public static final String TERM_URN_PREFIX = "urn:li:glossaryTerm:"; +} diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/models/FieldPathTest.java b/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/models/FieldPathTest.java new file mode 100644 index 00000000000000..d823a2c8ed51b7 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/test/java/io/datahubproject/schematron/models/FieldPathTest.java @@ -0,0 +1,246 @@ +package io.datahubproject.schematron.models; + +import static org.testng.Assert.*; + +import com.linkedin.schema.ArrayType; +import com.linkedin.schema.MapType; +import com.linkedin.schema.UnionType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.testng.annotations.*; + +@Test(groups = "unit") +public class FieldPathTest { + + @Test(groups = "basic") + public void testEmptyFieldPath() { + FieldPath path = new FieldPath(); + assertEquals(path.asString(), "[version=2.0]"); + } + + @Test(groups = "basic") + public void testKeySchemaPath() { + FieldPath path = new FieldPath(); + path.setKeySchema(true); + assertEquals(path.asString(), "[version=2.0].[key=True]"); + } + + @Test(groups = "basic") + public void testSimplePath() { + FieldPath path = new FieldPath(); + FieldElement element = + new FieldElement( + Collections.singletonList("string"), Collections.singletonList("schema"), "name", null); + path.setPath(Collections.singletonList(element)); + assertEquals(path.asString(), "[version=2.0].[type=string].name"); + } + + @Test(groups = "nested") + public void testNestedPath() { + FieldPath path = new FieldPath(); + FieldElement record = + new FieldElement( + Collections.singletonList("record"), + Collections.singletonList("record-schema"), + "user", + null); + FieldElement field = + new FieldElement( + Collections.singletonList("string"), + Collections.singletonList("string-schema"), + "name", + null); + path.setPath(Arrays.asList(record, field)); + assertEquals(path.asString(), "[version=2.0].[type=record].user.[type=string].name"); + } + + @Test(groups = "complex") + public void testUnionPath() { + FieldPath path = new FieldPath(); + + // Add union type + FieldElement union = + new FieldElement( + Collections.singletonList("union"), + Collections.singletonList("union-schema"), + "document", + null); + + // Add specific union member (record type) + FieldElement passport = + new FieldElement( + Collections.singletonList("Passport"), + Collections.singletonList("passport-schema"), + "document", + new DataHubType(UnionType.class, "Passport")); + + // Add field within the record + FieldElement number = + new FieldElement( + Collections.singletonList("string"), + Collections.singletonList("string-schema"), + "number", + null); + + path.setPath(Arrays.asList(union, passport, number)); + assertEquals( + path.asString(), + "[version=2.0].[type=union].document.[type=Passport].document.[type=string].number"); + } + + @Test(groups = "operations") + public void testClonePlus() { + FieldPath original = new FieldPath(); + FieldElement element1 = + new FieldElement( + Collections.singletonList("record"), + Collections.singletonList("schema1"), + "user", + null); + original.setPath(Collections.singletonList(element1)); + + FieldElement element2 = + new FieldElement( + Collections.singletonList("string"), + Collections.singletonList("schema2"), + "name", + null); + + FieldPath newPath = original.clonePlus(element2); + + // Verify original path remains unchanged + assertEquals(original.asString(), "[version=2.0].[type=record].user"); + + // Verify new path has both elements + assertEquals(newPath.asString(), "[version=2.0].[type=record].user.[type=string].name"); + } + + @Test(groups = "operations") + public void testExpandType() { + FieldPath path = new FieldPath(); + FieldElement element = new FieldElement(new ArrayList<>(), new ArrayList<>(), "field", null); + path.setPath(Collections.singletonList(element)); + + FieldPath expanded = path.expandType("string", "schema"); + + assertEquals(expanded.asString(), "[version=2.0].[type=string].field"); + assertEquals(expanded.getPath().get(0).getType().size(), 1); + assertEquals(expanded.getPath().get(0).getType().get(0), "string"); + assertEquals(expanded.getPath().get(0).getSchemaTypes().get(0), "schema"); + } + + @Test(groups = "operations") + public void testHasFieldName() { + FieldPath path = new FieldPath(); + assertFalse(path.hasFieldName()); + + FieldElement element = + new FieldElement( + Collections.singletonList("string"), Collections.singletonList("schema"), "name", null); + path.setPath(Collections.singletonList(element)); + assertTrue(path.hasFieldName()); + } + + @Test(groups = "operations") + public void testEnsureFieldName() { + FieldPath path = new FieldPath(); + assertFalse(path.hasFieldName()); + + path.ensureFieldName(); + assertTrue(path.hasFieldName()); + assertEquals(path.getPath().get(0).getName(), FieldPath.EMPTY_FIELD_NAME); + } + + @Test(groups = "complex") + public void testArrayPath() { + FieldPath path = new FieldPath(); + FieldElement array = + new FieldElement( + Collections.singletonList("array"), + Collections.singletonList("array-schema"), + "items", + new DataHubType(ArrayType.class, "string")); + + path.setPath(Collections.singletonList(array)); + assertEquals(path.asString(), "[version=2.0].[type=array].items"); + } + + @Test(groups = "complex") + public void testMapPath() { + FieldPath path = new FieldPath(); + FieldElement map = + new FieldElement( + Collections.singletonList("map"), + Collections.singletonList("map-schema"), + "properties", + new DataHubType(MapType.class, "string")); + + path.setPath(Collections.singletonList(map)); + assertEquals(path.asString(), "[version=2.0].[type=map].properties"); + } + + @Test(groups = "complex") + public void testMultipleTypesInPath() { + FieldPath path = new FieldPath(); + FieldElement element = + new FieldElement( + Arrays.asList("union", "string"), + Arrays.asList("union-schema", "string-schema"), + "field", + null); + path.setPath(Collections.singletonList(element)); + assertEquals(path.asString(), "[version=2.0].[type=union].[type=string].field"); + } + + @Test(groups = "complex") + public void testParentTypeHandling() { + FieldPath path = new FieldPath(); + DataHubType parentType = new DataHubType(ArrayType.class, "string"); + FieldElement element = + new FieldElement( + Collections.singletonList("array"), + Collections.singletonList("array-schema"), + "items", + parentType); + path.setPath(Collections.singletonList(element)); + + assertNotNull(path.getPath().get(0).getParentType()); + assertEquals(path.getPath().get(0).getParentType().getType(), ArrayType.class); + assertEquals(path.getPath().get(0).getParentType().getNestedType(), "string"); + } + + @Test(groups = "edge-cases") + public void testNoParentPath() { + FieldPath path = new FieldPath(); + assertEquals(path.asString(), "[version=2.0]"); + } + + @Test(groups = "edge-cases") + public void testEmptyElementList() { + FieldPath path = new FieldPath(); + path.setPath(new ArrayList<>()); + assertEquals(path.asString(), "[version=2.0]"); + } + + @DataProvider(name = "invalidPaths") + public Object[][] getInvalidPaths() { + return new Object[][] { + {null, "Expected IllegalArgumentException for null element"}, + { + Arrays.asList((FieldElement) null), + "Expected IllegalArgumentException for null element in list" + } + }; + } + + @Test( + groups = "edge-cases", + dataProvider = "invalidPaths", + expectedExceptions = IllegalArgumentException.class) + public void testInvalidPaths(List elements, String message) { + FieldPath path = new FieldPath(); + path.setPath(elements); + } +} diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile.avsc new file mode 100644 index 00000000000000..81f8b0e54b11e0 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile.avsc @@ -0,0 +1,456 @@ +{ + "type": "record", + "name": "CustomerProfile", + "namespace": "com.example.customer", + "doc": "A complex customer profile schema demonstrating various union types and optional fields", + "fields": [ + { + "name": "customerId", + "type": { + "type": "string", + "logicalType": "uuid" + }, + "doc": "Unique identifier for the customer" + }, + { + "name": "identificationDocument", + "type": [ + "null", + { + "type": "record", + "name": "Passport", + "fields": [ + { + "name": "passportNumber", + "type": "string" + }, + { + "name": "expiryDate", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "DriversLicense", + "fields": [ + { + "name": "licenseNumber", + "type": "string" + }, + { + "name": "state", + "type": "string" + }, + { + "name": "validUntil", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "NationalID", + "fields": [ + { + "name": "idNumber", + "type": "string" + }, + { + "name": "country", + "type": "string" + } + ] + } + ], + "default": null, + "doc": "Customer's identification document - can be passport, driver's license, or national ID" + }, + { + "name": "contactInfo", + "type": { + "type": "record", + "name": "ContactInformation", + "fields": [ + { + "name": "primaryContact", + "type": [ + { + "type": "record", + "name": "EmailContact", + "fields": [ + { + "name": "emailAddress", + "type": "string" + }, + { + "name": "isVerified", + "type": "boolean", + "default": false + } + ] + }, + { + "type": "record", + "name": "PhoneContact", + "fields": [ + { + "name": "countryCode", + "type": "string" + }, + { + "name": "number", + "type": "string" + }, + { + "name": "type", + "type": { + "type": "enum", + "name": "PhoneType", + "symbols": [ + "MOBILE", + "LANDLINE" + ] + } + } + ] + } + ], + "doc": "Primary contact method - either email or phone" + }, + { + "name": "alternativeContacts", + "type": { + "type": "array", + "items": [ + "null", + "EmailContact", + "PhoneContact" + ] + }, + "default": [], + "doc": "List of alternative contact methods" + } + ] + } + }, + { + "name": "addresses", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "Address", + "fields": [ + { + "name": "type", + "type": { + "type": "enum", + "name": "AddressType", + "symbols": [ + "RESIDENTIAL", + "BUSINESS", + "SHIPPING" + ] + }, + "default": "RESIDENTIAL" + }, + { + "name": "street", + "type": "string" + }, + { + "name": "city", + "type": "string" + }, + { + "name": "state", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "country", + "type": "string" + }, + { + "name": "postalCode", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "validationStatus", + "type": [ + "null", + { + "type": "record", + "name": "AddressValidation", + "fields": [ + { + "name": "isValid", + "type": "boolean" + }, + { + "name": "verificationDate", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + }, + { + "name": "verificationMethod", + "type": { + "type": "enum", + "name": "VerificationMethod", + "symbols": [ + "MANUAL", + "AUTOMATED" + ] + } + } + ] + } + ], + "default": null + } + ] + } + }, + "doc": "Customer's addresses with validation information" + }, + { + "name": "preferences", + "type": { + "type": "map", + "values": [ + "null", + "string", + "boolean", + { + "type": "record", + "name": "FrequencyPreference", + "fields": [ + { + "name": "frequency", + "type": { + "type": "enum", + "name": "Frequency", + "symbols": [ + "DAILY", + "WEEKLY", + "MONTHLY" + ] + } + }, + { + "name": "enabled", + "type": "boolean", + "default": true + }, + { + "name": "lastUpdated", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] + } + ] + }, + "doc": "Customer preferences with various possible value types" + }, + { + "name": "subscriptionHistory", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "Subscription", + "fields": [ + { + "name": "planName", + "type": "string" + }, + { + "name": "startDate", + "type": { + "type": "long", + "logicalType": "date" + } + }, + { + "name": "endDate", + "type": [ + "null", + { + "type": "long", + "logicalType": "date" + } + ], + "default": null + }, + { + "name": "status", + "type": { + "type": "enum", + "name": "SubscriptionStatus", + "symbols": [ + "ACTIVE", + "CANCELLED", + "EXPIRED", + "SUSPENDED" + ] + } + }, + { + "name": "paymentMethod", + "type": [ + "null", + { + "type": "record", + "name": "PaymentMethod", + "fields": [ + { + "name": "type", + "type": { + "type": "enum", + "name": "PaymentType", + "symbols": [ + "CREDIT_CARD", + "DEBIT_CARD", + "BANK_TRANSFER", + "DIGITAL_WALLET" + ] + } + }, + { + "name": "lastFourDigits", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "expiryDate", + "type": [ + "null", + { + "type": "long", + "logicalType": "date" + } + ], + "default": null + } + ] + } + ], + "default": null + } + ] + } + } + ], + "default": null, + "doc": "Historical record of customer subscriptions" + }, + { + "name": "metadata", + "type": { + "type": "map", + "values": [ + "null", + "string", + "long", + "boolean", + { + "type": "record", + "name": "MetadataValue", + "fields": [ + { + "name": "value", + "type": [ + "null", + "string", + "long", + "boolean" + ], + "default": null + }, + { + "name": "timestamp", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + }, + { + "name": "source", + "type": "string" + } + ] + } + ] + }, + "doc": "Flexible metadata storage with various possible value types" + }, + { + "name": "tags", + "type": [ + "null", + { + "type": "array", + "items": { + "type": "record", + "name": "Tag", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "value", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "score", + "type": [ + "null", + "double" + ], + "default": null + }, + { + "name": "addedAt", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] + } + } + ], + "default": null, + "doc": "Optional tags associated with the customer profile" + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile2.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile2.avsc new file mode 100644 index 00000000000000..b8c7654ea072a2 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/CustomerProfile2.avsc @@ -0,0 +1,244 @@ +{ + "type": "record", + "name": "CustomerProfile2", + "namespace": "com.example.customer", + "doc": "A complex customer profile schema demonstrating various union types and optional fields", + "fields": [ + { + "name": "customerId", + "type": { + "type": "string", + "logicalType": "uuid" + }, + "doc": "Unique identifier for the customer" + }, + { + "name": "identificationDocument", + "type": [ + "null", + { + "type": "record", + "name": "Passport", + "fields": [ + { + "name": "passportNumber", + "type": "string" + }, + { + "name": "expiryDate", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "DriversLicense", + "fields": [ + { + "name": "licenseNumber", + "type": "string" + }, + { + "name": "state", + "type": "string" + }, + { + "name": "validUntil", + "type": { + "type": "long", + "logicalType": "date" + } + } + ] + }, + { + "type": "record", + "name": "NationalID", + "fields": [ + { + "name": "idNumber", + "type": "string" + }, + { + "name": "country", + "type": "string" + } + ] + } + ], + "default": null, + "doc": "Customer's identification document" + }, + { + "name": "contactInfo", + "type": { + "type": "record", + "name": "ContactInformation", + "fields": [ + { + "name": "primaryEmailContact", + "type": [ + "null", + { + "type": "record", + "name": "PrimaryEmailContact", + "fields": [ + { + "name": "emailAddress", + "type": "string" + }, + { + "name": "isVerified", + "type": "boolean", + "default": false + } + ] + } + ], + "default": null + }, + { + "name": "primaryPhoneContact", + "type": [ + "null", + { + "type": "record", + "name": "PrimaryPhoneContact", + "fields": [ + { + "name": "countryCode", + "type": "string" + }, + { + "name": "number", + "type": "string" + }, + { + "name": "type", + "type": { + "type": "enum", + "name": "PhoneType", + "symbols": [ + "MOBILE", + "LANDLINE" + ] + } + } + ] + } + ], + "default": null + }, + { + "name": "alternativeEmailContacts", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "AlternativeEmailContact", + "fields": [ + { + "name": "emailAddress", + "type": "string" + }, + { + "name": "isVerified", + "type": "boolean", + "default": false + } + ] + } + }, + "default": [] + }, + { + "name": "alternativePhoneContacts", + "type": { + "type": "array", + "items": { + "type": "record", + "name": "AlternativePhoneContact", + "fields": [ + { + "name": "countryCode", + "type": "string" + }, + { + "name": "number", + "type": "string" + }, + { + "name": "type", + "type": "PhoneType" + } + ] + } + }, + "default": [] + } + ] + } + }, + { + "name": "preferences", + "type": { + "type": "record", + "name": "Preferences", + "fields": [ + { + "name": "simplePreferences", + "type": { + "type": "map", + "values": [ + "null", + "string", + "boolean" + ] + }, + "default": {} + }, + { + "name": "frequencyPreferences", + "type": { + "type": "map", + "values": { + "type": "record", + "name": "FrequencyPreference", + "fields": [ + { + "name": "frequency", + "type": { + "type": "enum", + "name": "Frequency", + "symbols": [ + "DAILY", + "WEEKLY", + "MONTHLY" + ] + } + }, + { + "name": "enabled", + "type": "boolean", + "default": true + }, + { + "name": "lastUpdated", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + } + } + ] + } + }, + "default": {} + } + ] + } + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/datahub-schematron/lib/src/test/resources/FlatUser.avsc b/metadata-integration/java/datahub-schematron/lib/src/test/resources/FlatUser.avsc new file mode 100644 index 00000000000000..c796878c32ae41 --- /dev/null +++ b/metadata-integration/java/datahub-schematron/lib/src/test/resources/FlatUser.avsc @@ -0,0 +1,45 @@ +{ + "type": "record", + "name": "FlatUser", + "namespace": "com.example", + "fields": [ + { + "name": "id", + "type": "int", + "doc": "The unique identifier for a user", + "default": -1, + "metadata": { + "key1": "value1", + "key2": "value2" + } + }, + { + "name": "username", + "type": "string", + "doc": "The username of the user" + }, + { + "name": "email", + "type": "string", + "doc": "The email of the user" + }, + { + "name": "age", + "type": "int", + "doc": "The age of the user" + }, + { + "name": "isActive", + "type": "boolean", + "doc": "Whether the user is active or not" + }, + { + "name": "registrationDate", + "type": { + "type": "long", + "logicalType": "timestamp-millis" + }, + "doc": "The registration date of the user" + } + ] +} \ No newline at end of file diff --git a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh index 854c4227d08d93..d4108421216489 100755 --- a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh +++ b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh @@ -41,7 +41,9 @@ jar -tvf $jarFile |\ grep -v "VersionInfo.java" |\ grep -v "mime.types" |\ grep -v "com/ibm/.*" |\ - grep -v "google/" + grep -v "google/" |\ + grep -v "org/apache/avro" |\ + grep -v "org/apache" if [ $? -ne 0 ]; then diff --git a/settings.gradle b/settings.gradle index fa1fdb9f1a67ce..8756df31c1ac6f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -75,3 +75,5 @@ include 'metadata-service:configuration' include ':metadata-jobs:common' include ':metadata-operation-context' include ':metadata-service:openapi-servlet:models' +include ':metadata-integration:java:datahub-schematron:lib' +include ':metadata-integration:java:datahub-schematron:cli' From 3b00fd76469bc971311a215f63f07c35b6b894f8 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 26 Nov 2024 21:02:28 -0600 Subject: [PATCH 080/174] fix(shadowJar): fix shadowJar (#11968) --- .github/workflows/check-datahub-jars.yml | 1 + .../java/datahub-protobuf/scripts/check_jar.sh | 4 +++- metadata-integration/java/datahub-schematron/lib/build.gradle | 4 ---- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml index becf8126dc45ba..7a49f32729ec1f 100644 --- a/.github/workflows/check-datahub-jars.yml +++ b/.github/workflows/check-datahub-jars.yml @@ -40,4 +40,5 @@ jobs: - name: check ${{ matrix.command }} jar run: | ./gradlew :metadata-integration:java:${{ matrix.command }}:build --info + ./gradlew :metadata-integration:java:${{ matrix.command }}:checkShadowJar ./gradlew :metadata-integration:java:${{ matrix.command }}:javadoc diff --git a/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh index bd0c28f0f86988..66c70f0b857692 100755 --- a/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh +++ b/metadata-integration/java/datahub-protobuf/scripts/check_jar.sh @@ -44,7 +44,9 @@ jar -tvf $jarFile |\ grep -v "mime.types" |\ grep -v "com/ibm/.*" |\ grep -v "org/glassfish/" |\ - grep -v "LICENSE" + grep -v "LICENSE" |\ + grep -v "org/apache/avro" |\ + grep -v "org/apache" if [ $? -ne 0 ]; then echo "✅ No unexpected class paths found in ${jarFile}" diff --git a/metadata-integration/java/datahub-schematron/lib/build.gradle b/metadata-integration/java/datahub-schematron/lib/build.gradle index 83dec1039f7be0..3ba22ff4cb7b5d 100644 --- a/metadata-integration/java/datahub-schematron/lib/build.gradle +++ b/metadata-integration/java/datahub-schematron/lib/build.gradle @@ -45,10 +45,6 @@ jacocoTestReport { test.finalizedBy jacocoTestReport -task checkShadowJar(type: Exec) { - commandLine 'sh', '-c', 'scripts/check_jar.sh' -} - configurations { provided implementation.extendsFrom provided From b6ccb8c9bf72608fa902e7ee694f3a19cb20f72f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 26 Nov 2024 22:20:50 -0500 Subject: [PATCH 081/174] fix(ingest): ensure sentry is initialized with graph tags (#11949) --- .../src/datahub/ingestion/graph/client.py | 2 ++ .../src/datahub/ingestion/run/pipeline.py | 9 +++--- .../src/datahub/telemetry/telemetry.py | 32 +++++++++++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index 759aebcfd46b0a..4aa937639e9590 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -67,6 +67,7 @@ SystemMetadataClass, TelemetryClientIdClass, ) +from datahub.telemetry.telemetry import telemetry_instance from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.str_enum import StrEnum from datahub.utilities.urns.urn import Urn, guess_entity_type @@ -1819,4 +1820,5 @@ def get_default_graph() -> DataHubGraph: graph_config = config_utils.load_client_config() graph = DataHubGraph(graph_config) graph.test_connection() + telemetry_instance.set_context(server=graph) return graph diff --git a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py index 7c3a42c3e08931..667129ff83584a 100644 --- a/metadata-ingestion/src/datahub/ingestion/run/pipeline.py +++ b/metadata-ingestion/src/datahub/ingestion/run/pipeline.py @@ -44,7 +44,8 @@ ) from datahub.ingestion.transformer.transform_registry import transform_registry from datahub.metadata.schema_classes import MetadataChangeProposalClass -from datahub.telemetry import stats, telemetry +from datahub.telemetry import stats +from datahub.telemetry.telemetry import telemetry_instance from datahub.utilities._custom_package_loader import model_version_name from datahub.utilities.global_warning_util import ( clear_global_warnings, @@ -273,8 +274,9 @@ def __init__( if self.graph is None and isinstance(self.sink, DatahubRestSink): with _add_init_error_context("setup default datahub client"): self.graph = self.sink.emitter.to_graph() + self.graph.test_connection() self.ctx.graph = self.graph - telemetry.telemetry_instance.update_capture_exception_context(server=self.graph) + telemetry_instance.set_context(server=self.graph) with set_graph_context(self.graph): with _add_init_error_context("configure reporters"): @@ -615,7 +617,7 @@ def log_ingestion_stats(self) -> None: sink_warnings = len(self.sink.get_report().warnings) global_warnings = len(get_global_warnings()) - telemetry.telemetry_instance.ping( + telemetry_instance.ping( "ingest_stats", { "source_type": self.source_type, @@ -637,7 +639,6 @@ def log_ingestion_stats(self) -> None: ), "has_pipeline_name": bool(self.config.pipeline_name), }, - self.ctx.graph, ) def _approx_all_vals(self, d: LossyList[Any]) -> int: diff --git a/metadata-ingestion/src/datahub/telemetry/telemetry.py b/metadata-ingestion/src/datahub/telemetry/telemetry.py index 4faf04ee2d2c76..22b2cb6a101af9 100644 --- a/metadata-ingestion/src/datahub/telemetry/telemetry.py +++ b/metadata-ingestion/src/datahub/telemetry/telemetry.py @@ -7,7 +7,7 @@ import uuid from functools import wraps from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, TypeVar from mixpanel import Consumer, Mixpanel from typing_extensions import ParamSpec @@ -16,10 +16,12 @@ from datahub.cli.config_utils import DATAHUB_ROOT_FOLDER from datahub.cli.env_utils import get_boolean_env_variable from datahub.configuration.common import ExceptionWithProps -from datahub.ingestion.graph.client import DataHubGraph from datahub.metadata.schema_classes import _custom_package_path from datahub.utilities.perf_timer import PerfTimer +if TYPE_CHECKING: + from datahub.ingestion.graph.client import DataHubGraph + logger = logging.getLogger(__name__) DATAHUB_FOLDER = Path(DATAHUB_ROOT_FOLDER) @@ -117,7 +119,11 @@ class Telemetry: tracking_init: bool = False sentry_enabled: bool = False + context_properties: Dict[str, Any] = {} + def __init__(self): + self.context_properties = {} + if SENTRY_DSN: self.sentry_enabled = True try: @@ -157,6 +163,9 @@ def __init__(self): except Exception as e: logger.debug(f"Error connecting to mixpanel: {e}") + # Initialize the default properties for all events. + self.set_context() + def update_config(self) -> bool: """ Update the config file with the current client ID and enabled status. @@ -238,18 +247,22 @@ def load_config(self) -> bool: return False - def update_capture_exception_context( + def set_context( self, - server: Optional[DataHubGraph] = None, + server: Optional["DataHubGraph"] = None, properties: Optional[Dict[str, Any]] = None, ) -> None: + self.context_properties = { + **self._server_props(server), + **(properties or {}), + } + if self.sentry_enabled: from sentry_sdk import set_tag properties = { **_default_telemetry_properties(), - **self._server_props(server), - **(properties or {}), + **self.context_properties, } for key in properties: @@ -297,7 +310,6 @@ def ping( self, event_name: str, properties: Optional[Dict[str, Any]] = None, - server: Optional[DataHubGraph] = None, ) -> None: """ Send a single telemetry event. @@ -323,14 +335,15 @@ def ping( properties = { **_default_telemetry_properties(), - **self._server_props(server), + **self.context_properties, **properties, } self.mp.track(self.client_id, event_name, properties) except Exception as e: logger.debug(f"Error reporting telemetry: {e}") - def _server_props(self, server: Optional[DataHubGraph]) -> Dict[str, str]: + @classmethod + def _server_props(cls, server: Optional["DataHubGraph"]) -> Dict[str, str]: if not server: return { "server_type": "n/a", @@ -435,6 +448,7 @@ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: **call_props, "status": "error", **_error_props(e), + "code": e.code, }, ) telemetry_instance.capture_exception(e) From 48d711b19873794a423d2bd63034e959a23615d0 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 27 Nov 2024 16:49:01 +0530 Subject: [PATCH 082/174] fix(ingest): more error handling (#11969) --- .../datahub/ingestion/source/gc/datahub_gc.py | 27 +++++++++++++++---- .../source/gc/dataprocess_cleanup.py | 4 ++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py index c4b4186f45fc38..52807ca2a3f026 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py @@ -144,15 +144,32 @@ def get_workunits_internal( self, ) -> Iterable[MetadataWorkUnit]: if self.config.cleanup_expired_tokens: - self.revoke_expired_tokens() + try: + self.revoke_expired_tokens() + except Exception as e: + self.report.failure("While trying to cleanup expired token ", exc=e) if self.config.truncate_indices: - self.truncate_indices() + try: + self.truncate_indices() + except Exception as e: + self.report.failure("While trying to truncate indices ", exc=e) if self.dataprocess_cleanup: - yield from self.dataprocess_cleanup.get_workunits_internal() + try: + yield from self.dataprocess_cleanup.get_workunits_internal() + except Exception as e: + self.report.failure("While trying to cleanup data process ", exc=e) if self.soft_deleted_entities_cleanup: - self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities() + try: + self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities() + except Exception as e: + self.report.failure( + "While trying to cleanup soft deleted entities ", exc=e + ) if self.execution_request_cleanup: - self.execution_request_cleanup.run() + try: + self.execution_request_cleanup.run() + except Exception as e: + self.report.failure("While trying to cleanup execution request ", exc=e) yield from [] def truncate_indices(self) -> None: diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 130f2c9c2e12fc..0f35e1a67fede7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -404,7 +404,9 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: try: self.delete_dpi_from_datajobs(datajob_entity) except Exception as e: - logger.error(f"While trying to delete {datajob_entity} got {e}") + self.report.failure( + f"While trying to delete {datajob_entity} ", exc=e + ) if ( datajob_entity.total_runs == 0 and self.config.delete_empty_data_jobs From 25493c1f718226aeb56557c166e801c3f46c6ae3 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 27 Nov 2024 07:56:15 -0600 Subject: [PATCH 083/174] feat(datahub-gc): add truncation days param (#11967) --- .../configuration/src/main/resources/bootstrap_mcps.yaml | 2 +- .../src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml index f9497258c384fc..0e283dfdfc93ca 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml @@ -38,7 +38,7 @@ bootstrap: # Ingestion Recipes - name: ingestion-datahub-gc - version: v4 + version: v5 optional: false mcps_location: "bootstrap_mcps/ingestion-datahub-gc.yaml" values_env: "DATAHUB_GC_BOOTSTRAP_VALUES" diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml index 395eb5db534245..c0c5be85b16b1d 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml @@ -19,6 +19,7 @@ config: cleanup_expired_tokens: {{cleanup_expired_tokens}}{{^cleanup_expired_tokens}}false{{/cleanup_expired_tokens}} truncate_indices: {{truncate_indices}}{{^truncate_indices}}true{{/truncate_indices}} + truncate_index_older_than_days: {{truncate_indices_retention_days}}{{^truncate_indices_retention_days}}30{{/truncate_indices_retention_days}} dataprocess_cleanup: retention_days: {{dataprocess_cleanup.retention_days}}{{^dataprocess_cleanup.retention_days}}10{{/dataprocess_cleanup.retention_days}} delete_empty_data_jobs: {{dataprocess_cleanup.delete_empty_data_jobs}}{{^dataprocess_cleanup.delete_empty_data_jobs}}true{{/dataprocess_cleanup.delete_empty_data_jobs}} From 0b50f6f8c3505aee7359b94cb25d4585cdbf60dc Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:44:59 -0600 Subject: [PATCH 084/174] docs(release): Update v_0_3_7.md (#11937) Co-authored-by: John Joyce --- docs/managed-datahub/release-notes/v_0_3_7.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index af23b5ae1541b5..694283d945facc 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -19,6 +19,26 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies ## Release Changelog --- +### v0.3.7.4 + +- [#11935](https://github.com/datahub-project/datahub/pull/11935) - Added environment variable for enabling stricter URN validation rules `STRICT_URN_VALIDATION_ENABLED` [[1](https://datahubproject.io/docs/what/urn/#restrictions)]. +- [Automations] Filter out self-nodes in glossary term propagation +- [Remote Executor] Allow dashes in executor ids. +- [Search] Fix Nested Filter Counts in Primary Search +- [Search] Fix white screen of death on empty search result +- [Columns Tab] Support searching nested struct columns correctly in V2 UI. +- [Logo] Fix fit of custom logo for V2 UI nav bar. +- [Structured Properties] Better handling for special characters in structured properties +- [Lineage] Improvements to handling lineage cycles +- [Metadata Tests] Improve Reliability of Metadata Tests Action Application +- [Slack Integration] Minor improvement in authentication redirect to integrate with Slack +- [Columns Tab] Property display nullable status in column sidebar (bug) +- [Columns Tab] Fixing merging of sibling schemas between V2 and V1 field paths. +- [Documentation] Support group authors for institutional memory aspect + + +### v0.3.7 + - All changes in https://github.com/datahub-project/datahub/releases/tag/v0.14.1 - Note Breaking Changes: https://datahubproject.io/docs/how/updating-datahub/#0141 From 36afa5cfa3856aeb6dccf742d9ad51303da3db26 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 27 Nov 2024 10:54:43 -0600 Subject: [PATCH 085/174] fix(ci): fix build-and-test (#11974) --- .github/workflows/build-and-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 532ba1102ed579..412c962cb6e36f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -83,6 +83,7 @@ jobs: - uses: gradle/actions/setup-gradle@v3 - name: Gradle build (and test) for NOT metadata ingestion if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} + # datahub-schematron:cli excluded due to dependency on metadata-ingestion run: | ./gradlew build \ -x :metadata-ingestion:build \ @@ -100,6 +101,7 @@ jobs: -x :metadata-ingestion-modules:gx-plugin:check \ -x :datahub-frontend:build \ -x :datahub-web-react:build \ + -x :metadata-integration:java:datahub-schematron:cli:test \ --parallel - name: Gradle build (and test) for frontend if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }} From 7bf767373512d661a23bb19b3fd623d407c8950c Mon Sep 17 00:00:00 2001 From: sid-acryl <155424659+sid-acryl@users.noreply.github.com> Date: Wed, 27 Nov 2024 23:02:24 +0530 Subject: [PATCH 086/174] refactor(ingest/powerbi): organize code within the module based on responsibilities (#11924) --- metadata-ingestion/setup.py | 2 +- .../ingestion/source/powerbi/__init__.py | 1 - .../ingestion/source/powerbi/config.py | 6 +- .../source/powerbi/m_query/data_classes.py | 36 +- .../source/powerbi/m_query/parser.py | 9 +- .../source/powerbi/m_query/pattern_handler.py | 920 +++++++++++++++++ .../source/powerbi/m_query/resolver.py | 954 +----------------- .../source/powerbi/m_query/validator.py | 12 +- .../ingestion/source/powerbi/powerbi.py | 18 +- .../integration/powerbi/test_m_parser.py | 52 +- .../tests/unit/test_powerbi_parser.py | 6 +- 11 files changed, 1042 insertions(+), 974 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 8ae112c0ab0b2b..74c2e611cf68f1 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -773,7 +773,7 @@ "trino = datahub.ingestion.source.sql.trino:TrinoSource", "starburst-trino-usage = datahub.ingestion.source.usage.starburst_trino_usage:TrinoUsageSource", "nifi = datahub.ingestion.source.nifi:NifiSource", - "powerbi = datahub.ingestion.source.powerbi:PowerBiDashboardSource", + "powerbi = datahub.ingestion.source.powerbi.powerbi:PowerBiDashboardSource", "powerbi-report-server = datahub.ingestion.source.powerbi_report_server:PowerBiReportServerDashboardSource", "iceberg = datahub.ingestion.source.iceberg.iceberg:IcebergSource", "vertica = datahub.ingestion.source.sql.vertica:VerticaSource", diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py index 1068f335e8f8e5..e69de29bb2d1d6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/__init__.py @@ -1 +0,0 @@ -from datahub.ingestion.source.powerbi.powerbi import PowerBiDashboardSource diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index 91fa2e96be2cce..f7458c4eb4d5b5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -173,7 +173,7 @@ class SupportedDataPlatform(Enum): datahub_data_platform_name="redshift", ) - DATABRICK_SQL = DataPlatformPair( + DATABRICKS_SQL = DataPlatformPair( powerbi_data_platform_name="Databricks", datahub_data_platform_name="databricks" ) @@ -313,8 +313,8 @@ class PowerBiDashboardSourceConfig( " Note: This field works in conjunction with 'workspace_type_filter' and both must be considered when filtering workspaces.", ) - # Dataset type mapping PowerBI support many type of data-sources. Here user need to define what type of PowerBI - # DataSource need to be mapped to corresponding DataHub Platform DataSource. For example PowerBI `Snowflake` is + # Dataset type mapping PowerBI support many type of data-sources. Here user needs to define what type of PowerBI + # DataSource needs to be mapped to corresponding DataHub Platform DataSource. For example, PowerBI `Snowflake` is # mapped to DataHub `snowflake` PowerBI `PostgreSQL` is mapped to DataHub `postgres` and so on. dataset_type_mapping: Union[ Dict[str, str], Dict[str, PlatformDetail] diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py index bb0c0c2f79bbdd..f1691b5df68a94 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py @@ -1,10 +1,14 @@ import os from abc import ABC from dataclasses import dataclass -from typing import Any, Dict, Optional +from enum import Enum +from typing import Any, Dict, List, Optional from lark import Tree +from datahub.ingestion.source.powerbi.config import DataPlatformPair +from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo + TRACE_POWERBI_MQUERY_PARSER = os.getenv("DATAHUB_TRACE_POWERBI_MQUERY_PARSER", False) @@ -30,7 +34,7 @@ class IdentifierAccessor(AbstractIdentifierAccessor): "[Schema="public",Item="order_date"]" is "items" in ItemSelector. Data of items varies as per DataSource - "public_order_date" is in "next" of ItemSelector. The "next" will be None if this identifier is leaf i.e. table + "public_order_date" is in "next" of ItemSelector. The "next" will be None if this identifier is leaf i.e., table """ @@ -53,3 +57,31 @@ class ReferencedTable: database: str schema: str table: str + + +@dataclass +class DataPlatformTable: + data_platform_pair: DataPlatformPair + urn: str + + +@dataclass +class Lineage: + upstreams: List[DataPlatformTable] + column_lineage: List[ColumnLineageInfo] + + @staticmethod + def empty() -> "Lineage": + return Lineage(upstreams=[], column_lineage=[]) + + +class FunctionName(Enum): + NATIVE_QUERY = "Value.NativeQuery" + POSTGRESQL_DATA_ACCESS = "PostgreSQL.Database" + ORACLE_DATA_ACCESS = "Oracle.Database" + SNOWFLAKE_DATA_ACCESS = "Snowflake.Databases" + MSSQL_DATA_ACCESS = "Sql.Database" + DATABRICK_DATA_ACCESS = "Databricks.Catalogs" + GOOGLE_BIGQUERY_DATA_ACCESS = "GoogleBigQuery.Database" + AMAZON_REDSHIFT_DATA_ACCESS = "AmazonRedshift.Database" + DATABRICK_MULTI_CLOUD_DATA_ACCESS = "DatabricksMultiCloud.Catalogs" diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py index 97698a3d0d56c1..2a5de7494920b2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/parser.py @@ -7,6 +7,7 @@ import lark from lark import Lark, Tree +import datahub.ingestion.source.powerbi.m_query.data_classes from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.powerbi.config import ( PowerBiDashboardSourceConfig, @@ -65,7 +66,7 @@ def get_upstream_tables( ctx: PipelineContext, config: PowerBiDashboardSourceConfig, parameters: Dict[str, str] = {}, -) -> List[resolver.Lineage]: +) -> List[datahub.ingestion.source.powerbi.m_query.data_classes.Lineage]: if table.expression is None: logger.debug(f"There is no M-Query expression in table {table.full_name}") return [] @@ -127,12 +128,14 @@ def get_upstream_tables( reporter.m_query_parse_successes += 1 try: - lineage: List[resolver.Lineage] = resolver.MQueryResolver( + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = resolver.MQueryResolver( table=table, parse_tree=parse_tree, reporter=reporter, parameters=parameters, - ).resolve_to_data_platform_table_list( + ).resolve_to_lineage( ctx=ctx, config=config, platform_instance_resolver=platform_instance_resolver, diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py new file mode 100644 index 00000000000000..13d97a70290298 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py @@ -0,0 +1,920 @@ +import logging +from abc import ABC, abstractmethod +from enum import Enum +from typing import Dict, List, Optional, Tuple, Type, Union, cast + +from lark import Tree + +from datahub.emitter import mce_builder as builder +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.powerbi.config import ( + Constant, + DataBricksPlatformDetail, + DataPlatformPair, + PlatformDetail, + PowerBiDashboardSourceConfig, + PowerBiDashboardSourceReport, + PowerBIPlatformDetail, + SupportedDataPlatform, +) +from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import ( + AbstractDataPlatformInstanceResolver, +) +from datahub.ingestion.source.powerbi.m_query import native_sql_parser, tree_function +from datahub.ingestion.source.powerbi.m_query.data_classes import ( + AbstractIdentifierAccessor, + DataAccessFunctionDetail, + DataPlatformTable, + FunctionName, + IdentifierAccessor, + Lineage, + ReferencedTable, +) +from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table +from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult + +logger = logging.getLogger(__name__) + + +def get_next_item(items: List[str], item: str) -> Optional[str]: + if item in items: + try: + index = items.index(item) + return items[index + 1] + except IndexError: + logger.debug(f'item:"{item}", not found in item-list: {items}') + return None + + +def urn_to_lowercase(value: str, flag: bool) -> str: + if flag is True: + return value.lower() + + return value + + +def make_urn( + config: PowerBiDashboardSourceConfig, + platform_instance_resolver: AbstractDataPlatformInstanceResolver, + data_platform_pair: DataPlatformPair, + server: str, + qualified_table_name: str, +) -> str: + platform_detail: PlatformDetail = platform_instance_resolver.get_platform_instance( + PowerBIPlatformDetail( + data_platform_pair=data_platform_pair, + data_platform_server=server, + ) + ) + + return builder.make_dataset_urn_with_platform_instance( + platform=data_platform_pair.datahub_data_platform_name, + platform_instance=platform_detail.platform_instance, + env=platform_detail.env, + name=urn_to_lowercase( + qualified_table_name, config.convert_lineage_urns_to_lowercase + ), + ) + + +class AbstractLineage(ABC): + """ + Base class to share common functionalities among different dataplatform for M-Query parsing. + + To create qualified table name we need to parse M-Query data-access-functions(https://learn.microsoft.com/en-us/powerquery-m/accessing-data-functions) and + the data-access-functions has some define pattern to access database-name, schema-name and table-name, for example, see below M-Query. + + let + Source = Sql.Database("localhost", "library"), + dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data] + in + dbo_book_issue + + It is MSSQL M-Query and Sql.Database is the data-access-function to access MSSQL. If this function is available in M-Query then database name is available in the second argument of the first statement and schema-name and table-name is available in the second statement. the second statement can be repeated to access different tables from MSSQL. + + DefaultTwoStepDataAccessSources extends the AbstractDataPlatformTableCreator and provides the common functionalities for data-platform which has above type of M-Query pattern + + data-access-function varies as per data-platform for example for MySQL.Database for MySQL, PostgreSQL.Database for Postgres and Oracle.Database for Oracle and number of statement to + find out database-name , schema-name and table-name also varies as per dataplatform. + + Value.NativeQuery is one of the functions which is used to execute a native query inside M-Query, for example see below M-Query + + let + Source = Value.NativeQuery(AmazonRedshift.Database("redshift-url","dev"), "select * from dev.public.category", null, [EnableFolding=true]) + in + Source + + In this M-Query database-name is available in first argument and rest of the detail i.e database & schema is available in native query. + + NativeQueryDataPlatformTableCreator extends AbstractDataPlatformTableCreator to support Redshift and Snowflake native query parsing. + + """ + + ctx: PipelineContext + table: Table + config: PowerBiDashboardSourceConfig + reporter: PowerBiDashboardSourceReport + platform_instance_resolver: AbstractDataPlatformInstanceResolver + + def __init__( + self, + ctx: PipelineContext, + table: Table, + config: PowerBiDashboardSourceConfig, + reporter: PowerBiDashboardSourceReport, + platform_instance_resolver: AbstractDataPlatformInstanceResolver, + ) -> None: + super().__init__() + self.ctx = ctx + self.table = table + self.config = config + self.reporter = reporter + self.platform_instance_resolver = platform_instance_resolver + + @abstractmethod + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + pass + + @abstractmethod + def get_platform_pair(self) -> DataPlatformPair: + pass + + @staticmethod + def get_db_detail_from_argument( + arg_list: Tree, + ) -> Tuple[Optional[str], Optional[str]]: + arguments: List[str] = tree_function.strip_char_from_list( + values=tree_function.remove_whitespaces_from_list( + tree_function.token_values(arg_list) + ), + ) + + if len(arguments) < 2: + logger.debug(f"Expected minimum 2 arguments, but got {len(arguments)}") + return None, None + + return arguments[0], arguments[1] + + @staticmethod + def create_reference_table( + arg_list: Tree, + table_detail: Dict[str, str], + ) -> Optional[ReferencedTable]: + arguments: List[str] = tree_function.strip_char_from_list( + values=tree_function.remove_whitespaces_from_list( + tree_function.token_values(arg_list) + ), + ) + + logger.debug(f"Processing arguments {arguments}") + + if ( + len(arguments) + >= 4 # [0] is warehouse FQDN. + # [1] is endpoint, we are not using it. + # [2] is "Catalog" key + # [3] is catalog's value + ): + return ReferencedTable( + warehouse=arguments[0], + catalog=arguments[3], + # As per my observation, database and catalog names are same in M-Query + database=table_detail["Database"] + if table_detail.get("Database") + else arguments[3], + schema=table_detail["Schema"], + table=table_detail.get("Table") or table_detail["View"], + ) + elif len(arguments) == 2: + return ReferencedTable( + warehouse=arguments[0], + database=table_detail["Database"], + schema=table_detail["Schema"], + table=table_detail.get("Table") or table_detail["View"], + catalog=None, + ) + + return None + + def parse_custom_sql( + self, query: str, server: str, database: Optional[str], schema: Optional[str] + ) -> Lineage: + dataplatform_tables: List[DataPlatformTable] = [] + + platform_detail: PlatformDetail = ( + self.platform_instance_resolver.get_platform_instance( + PowerBIPlatformDetail( + data_platform_pair=self.get_platform_pair(), + data_platform_server=server, + ) + ) + ) + + query = native_sql_parser.remove_drop_statement( + native_sql_parser.remove_special_characters(query) + ) + + parsed_result: Optional[ + "SqlParsingResult" + ] = native_sql_parser.parse_custom_sql( + ctx=self.ctx, + query=query, + platform=self.get_platform_pair().datahub_data_platform_name, + platform_instance=platform_detail.platform_instance, + env=platform_detail.env, + database=database, + schema=schema, + ) + + if parsed_result is None: + self.reporter.info( + title=Constant.SQL_PARSING_FAILURE, + message="Fail to parse native sql present in PowerBI M-Query", + context=f"table-name={self.table.full_name}, sql={query}", + ) + return Lineage.empty() + + if parsed_result.debug_info and parsed_result.debug_info.table_error: + self.reporter.warning( + title=Constant.SQL_PARSING_FAILURE, + message="Fail to parse native sql present in PowerBI M-Query", + context=f"table-name={self.table.full_name}, error={parsed_result.debug_info.table_error},sql={query}", + ) + return Lineage.empty() + + for urn in parsed_result.in_tables: + dataplatform_tables.append( + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ) + + logger.debug(f"Native Query parsed result={parsed_result}") + logger.debug(f"Generated dataplatform_tables={dataplatform_tables}") + + return Lineage( + upstreams=dataplatform_tables, + column_lineage=( + parsed_result.column_lineage + if parsed_result.column_lineage is not None + else [] + ), + ) + + +class AmazonRedshiftLineage(AbstractLineage): + def get_platform_pair(self) -> DataPlatformPair: + return SupportedDataPlatform.AMAZON_REDSHIFT.value + + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + logger.debug( + f"Processing AmazonRedshift data-access function detail {data_access_func_detail}" + ) + + server, db_name = self.get_db_detail_from_argument( + data_access_func_detail.arg_list + ) + if db_name is None or server is None: + return Lineage.empty() # Return an empty list + + schema_name: str = cast( + IdentifierAccessor, data_access_func_detail.identifier_accessor + ).items["Name"] + + table_name: str = cast( + IdentifierAccessor, + cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next, + ).items["Name"] + + qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" + + urn = make_urn( + config=self.config, + platform_instance_resolver=self.platform_instance_resolver, + data_platform_pair=self.get_platform_pair(), + server=server, + qualified_table_name=qualified_table_name, + ) + + return Lineage( + upstreams=[ + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ], + column_lineage=[], + ) + + +class OracleLineage(AbstractLineage): + def get_platform_pair(self) -> DataPlatformPair: + return SupportedDataPlatform.ORACLE.value + + @staticmethod + def _get_server_and_db_name(value: str) -> Tuple[Optional[str], Optional[str]]: + error_message: str = ( + f"The target argument ({value}) should in the format of :/[" + ".]" + ) + splitter_result: List[str] = value.split("/") + if len(splitter_result) != 2: + logger.debug(error_message) + return None, None + + db_name = splitter_result[1].split(".")[0] + + return tree_function.strip_char_from_list([splitter_result[0]])[0], db_name + + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + logger.debug( + f"Processing Oracle data-access function detail {data_access_func_detail}" + ) + + arguments: List[str] = tree_function.remove_whitespaces_from_list( + tree_function.token_values(data_access_func_detail.arg_list) + ) + + server, db_name = self._get_server_and_db_name(arguments[0]) + + if db_name is None or server is None: + return Lineage.empty() + + schema_name: str = cast( + IdentifierAccessor, data_access_func_detail.identifier_accessor + ).items["Schema"] + + table_name: str = cast( + IdentifierAccessor, + cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next, + ).items["Name"] + + qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" + + urn = make_urn( + config=self.config, + platform_instance_resolver=self.platform_instance_resolver, + data_platform_pair=self.get_platform_pair(), + server=server, + qualified_table_name=qualified_table_name, + ) + + return Lineage( + upstreams=[ + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ], + column_lineage=[], + ) + + +class DatabricksLineage(AbstractLineage): + def form_qualified_table_name( + self, + table_reference: ReferencedTable, + data_platform_pair: DataPlatformPair, + ) -> str: + platform_detail: PlatformDetail = ( + self.platform_instance_resolver.get_platform_instance( + PowerBIPlatformDetail( + data_platform_pair=data_platform_pair, + data_platform_server=table_reference.warehouse, + ) + ) + ) + + metastore: Optional[str] = None + + qualified_table_name: str = f"{table_reference.database}.{table_reference.schema}.{table_reference.table}" + + if isinstance(platform_detail, DataBricksPlatformDetail): + metastore = platform_detail.metastore + + if metastore is not None: + return f"{metastore}.{qualified_table_name}" + + return qualified_table_name + + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + logger.debug( + f"Processing Databrick data-access function detail {data_access_func_detail}" + ) + table_detail: Dict[str, str] = {} + temp_accessor: Optional[ + Union[IdentifierAccessor, AbstractIdentifierAccessor] + ] = data_access_func_detail.identifier_accessor + + while temp_accessor: + if isinstance(temp_accessor, IdentifierAccessor): + # Condition to handle databricks M-query pattern where table, schema and database all are present in + # the same invoke statement + if all( + element in temp_accessor.items + for element in ["Item", "Schema", "Catalog"] + ): + table_detail["Schema"] = temp_accessor.items["Schema"] + table_detail["Table"] = temp_accessor.items["Item"] + else: + table_detail[temp_accessor.items["Kind"]] = temp_accessor.items[ + "Name" + ] + + if temp_accessor.next is not None: + temp_accessor = temp_accessor.next + else: + break + else: + logger.debug( + "expecting instance to be IdentifierAccessor, please check if parsing is done properly" + ) + return Lineage.empty() + + table_reference = self.create_reference_table( + arg_list=data_access_func_detail.arg_list, + table_detail=table_detail, + ) + + if table_reference: + qualified_table_name: str = self.form_qualified_table_name( + table_reference=table_reference, + data_platform_pair=self.get_platform_pair(), + ) + + urn = make_urn( + config=self.config, + platform_instance_resolver=self.platform_instance_resolver, + data_platform_pair=self.get_platform_pair(), + server=table_reference.warehouse, + qualified_table_name=qualified_table_name, + ) + + return Lineage( + upstreams=[ + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ], + column_lineage=[], + ) + + return Lineage.empty() + + def get_platform_pair(self) -> DataPlatformPair: + return SupportedDataPlatform.DATABRICKS_SQL.value + + +class TwoStepDataAccessPattern(AbstractLineage, ABC): + """ + These are the DataSource for which PowerBI Desktop generates default M-Query of the following pattern + let + Source = Sql.Database("localhost", "library"), + dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data] + in + dbo_book_issue + """ + + def two_level_access_pattern( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + logger.debug( + f"Processing {self.get_platform_pair().powerbi_data_platform_name} data-access function detail {data_access_func_detail}" + ) + + server, db_name = self.get_db_detail_from_argument( + data_access_func_detail.arg_list + ) + if server is None or db_name is None: + return Lineage.empty() # Return an empty list + + schema_name: str = cast( + IdentifierAccessor, data_access_func_detail.identifier_accessor + ).items["Schema"] + + table_name: str = cast( + IdentifierAccessor, data_access_func_detail.identifier_accessor + ).items["Item"] + + qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" + + logger.debug( + f"Platform({self.get_platform_pair().datahub_data_platform_name}) qualified_table_name= {qualified_table_name}" + ) + + urn = make_urn( + config=self.config, + platform_instance_resolver=self.platform_instance_resolver, + data_platform_pair=self.get_platform_pair(), + server=server, + qualified_table_name=qualified_table_name, + ) + return Lineage( + upstreams=[ + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ], + column_lineage=[], + ) + + +class PostgresLineage(TwoStepDataAccessPattern): + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + return self.two_level_access_pattern(data_access_func_detail) + + def get_platform_pair(self) -> DataPlatformPair: + return SupportedDataPlatform.POSTGRES_SQL.value + + +class MSSqlLineage(TwoStepDataAccessPattern): + # https://learn.microsoft.com/en-us/sql/relational-databases/security/authentication-access/ownership-and-user-schema-separation?view=sql-server-ver16 + DEFAULT_SCHEMA = "dbo" # Default schema name in MS-SQL is dbo + + def get_platform_pair(self) -> DataPlatformPair: + return SupportedDataPlatform.MS_SQL.value + + def create_urn_using_old_parser( + self, query: str, db_name: str, server: str + ) -> List[DataPlatformTable]: + dataplatform_tables: List[DataPlatformTable] = [] + + tables: List[str] = native_sql_parser.get_tables(query) + + for parsed_table in tables: + # components: List[str] = [v.strip("[]") for v in parsed_table.split(".")] + components = [v.strip("[]") for v in parsed_table.split(".")] + if len(components) == 3: + database, schema, table = components + elif len(components) == 2: + schema, table = components + database = db_name + elif len(components) == 1: + (table,) = components + database = db_name + schema = MSSqlLineage.DEFAULT_SCHEMA + else: + self.reporter.warning( + title="Invalid table format", + message="The advanced SQL lineage feature (enable_advance_lineage_sql_construct) is disabled. Please either enable this feature or ensure the table is referenced as .. in the SQL.", + context=f"table-name={self.table.full_name}", + ) + continue + + qualified_table_name = f"{database}.{schema}.{table}" + urn = make_urn( + config=self.config, + platform_instance_resolver=self.platform_instance_resolver, + data_platform_pair=self.get_platform_pair(), + server=server, + qualified_table_name=qualified_table_name, + ) + dataplatform_tables.append( + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ) + + logger.debug(f"Generated upstream tables = {dataplatform_tables}") + + return dataplatform_tables + + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + arguments: List[str] = tree_function.strip_char_from_list( + values=tree_function.remove_whitespaces_from_list( + tree_function.token_values(data_access_func_detail.arg_list) + ), + ) + + server, database = self.get_db_detail_from_argument( + data_access_func_detail.arg_list + ) + if server is None or database is None: + return Lineage.empty() # Return an empty list + + assert server + assert database # to silent the lint + + query: Optional[str] = get_next_item(arguments, "Query") + if query: + if self.config.enable_advance_lineage_sql_construct is False: + # Use previous parser to generate URN to keep backward compatibility + return Lineage( + upstreams=self.create_urn_using_old_parser( + query=query, + db_name=database, + server=server, + ), + column_lineage=[], + ) + + return self.parse_custom_sql( + query=query, + database=database, + server=server, + schema=MSSqlLineage.DEFAULT_SCHEMA, + ) + + # It is a regular case of MS-SQL + logger.debug("Handling with regular case") + return self.two_level_access_pattern(data_access_func_detail) + + +class ThreeStepDataAccessPattern(AbstractLineage, ABC): + def get_datasource_server( + self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail + ) -> str: + return tree_function.strip_char_from_list([arguments[0]])[0] + + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + logger.debug( + f"Processing {self.get_platform_pair().datahub_data_platform_name} function detail {data_access_func_detail}" + ) + + arguments: List[str] = tree_function.remove_whitespaces_from_list( + tree_function.token_values(data_access_func_detail.arg_list) + ) + # First is database name + db_name: str = data_access_func_detail.identifier_accessor.items["Name"] # type: ignore + # Second is schema name + schema_name: str = cast( + IdentifierAccessor, data_access_func_detail.identifier_accessor.next # type: ignore + ).items["Name"] + # Third is table name + table_name: str = cast( + IdentifierAccessor, data_access_func_detail.identifier_accessor.next.next # type: ignore + ).items["Name"] + + qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" + + logger.debug( + f"{self.get_platform_pair().datahub_data_platform_name} qualified_table_name {qualified_table_name}" + ) + + server: str = self.get_datasource_server(arguments, data_access_func_detail) + + urn = make_urn( + config=self.config, + platform_instance_resolver=self.platform_instance_resolver, + data_platform_pair=self.get_platform_pair(), + server=server, + qualified_table_name=qualified_table_name, + ) + + return Lineage( + upstreams=[ + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ], + column_lineage=[], + ) + + +class SnowflakeLineage(ThreeStepDataAccessPattern): + def get_platform_pair(self) -> DataPlatformPair: + return SupportedDataPlatform.SNOWFLAKE.value + + +class GoogleBigQueryLineage(ThreeStepDataAccessPattern): + def get_platform_pair(self) -> DataPlatformPair: + return SupportedDataPlatform.GOOGLE_BIGQUERY.value + + def get_datasource_server( + self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail + ) -> str: + # In Google BigQuery server is project-name + # condition to silent lint, it is not going to be None + return ( + data_access_func_detail.identifier_accessor.items["Name"] + if data_access_func_detail.identifier_accessor is not None + else "" + ) + + +class NativeQueryLineage(AbstractLineage): + SUPPORTED_NATIVE_QUERY_DATA_PLATFORM: dict = { + SupportedDataPlatform.SNOWFLAKE.value.powerbi_data_platform_name: SupportedDataPlatform.SNOWFLAKE, + SupportedDataPlatform.AMAZON_REDSHIFT.value.powerbi_data_platform_name: SupportedDataPlatform.AMAZON_REDSHIFT, + SupportedDataPlatform.DatabricksMultiCloud_SQL.value.powerbi_data_platform_name: SupportedDataPlatform.DatabricksMultiCloud_SQL, + } + current_data_platform: SupportedDataPlatform = SupportedDataPlatform.SNOWFLAKE + + def get_platform_pair(self) -> DataPlatformPair: + return self.current_data_platform.value + + @staticmethod + def is_native_parsing_supported(data_access_function_name: str) -> bool: + return ( + data_access_function_name + in NativeQueryLineage.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM + ) + + def create_urn_using_old_parser(self, query: str, server: str) -> Lineage: + dataplatform_tables: List[DataPlatformTable] = [] + + tables: List[str] = native_sql_parser.get_tables(query) + + for qualified_table_name in tables: + if len(qualified_table_name.split(".")) != 3: + logger.debug( + f"Skipping table {qualified_table_name} as it is not as per qualified_table_name format" + ) + continue + + urn = make_urn( + config=self.config, + platform_instance_resolver=self.platform_instance_resolver, + data_platform_pair=self.get_platform_pair(), + server=server, + qualified_table_name=qualified_table_name, + ) + + dataplatform_tables.append( + DataPlatformTable( + data_platform_pair=self.get_platform_pair(), + urn=urn, + ) + ) + + logger.debug(f"Generated dataplatform_tables {dataplatform_tables}") + + return Lineage( + upstreams=dataplatform_tables, + column_lineage=[], + ) + + def get_db_name(self, data_access_tokens: List[str]) -> Optional[str]: + if ( + data_access_tokens[0] + != SupportedDataPlatform.DatabricksMultiCloud_SQL.value.powerbi_data_platform_name + ): + return None + + database: Optional[str] = get_next_item(data_access_tokens, "Database") + + if ( + database and database != Constant.M_QUERY_NULL + ): # database name is explicitly set + return database + + return get_next_item( # database name is set in Name argument + data_access_tokens, "Name" + ) or get_next_item( # If both above arguments are not available, then try Catalog + data_access_tokens, "Catalog" + ) + + def create_lineage( + self, data_access_func_detail: DataAccessFunctionDetail + ) -> Lineage: + t1: Tree = cast( + Tree, tree_function.first_arg_list_func(data_access_func_detail.arg_list) + ) + flat_argument_list: List[Tree] = tree_function.flat_argument_list(t1) + + if len(flat_argument_list) != 2: + logger.debug( + f"Expecting 2 argument, actual argument count is {len(flat_argument_list)}" + ) + logger.debug(f"Flat argument list = {flat_argument_list}") + return Lineage.empty() + + data_access_tokens: List[str] = tree_function.remove_whitespaces_from_list( + tree_function.token_values(flat_argument_list[0]) + ) + + if not self.is_native_parsing_supported(data_access_tokens[0]): + logger.debug( + f"Unsupported native-query data-platform = {data_access_tokens[0]}" + ) + logger.debug( + f"NativeQuery is supported only for {self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM}" + ) + + return Lineage.empty() + + if len(data_access_tokens[0]) < 3: + logger.debug( + f"Server is not available in argument list for data-platform {data_access_tokens[0]}. Returning empty " + "list" + ) + return Lineage.empty() + + self.current_data_platform = self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM[ + data_access_tokens[0] + ] + # The First argument is the query + sql_query: str = tree_function.strip_char_from_list( + values=tree_function.remove_whitespaces_from_list( + tree_function.token_values(flat_argument_list[1]) + ), + )[ + 0 + ] # Remove any whitespaces and double quotes character + + server = tree_function.strip_char_from_list([data_access_tokens[2]])[0] + + if self.config.enable_advance_lineage_sql_construct is False: + # Use previous parser to generate URN to keep backward compatibility + return self.create_urn_using_old_parser( + query=sql_query, + server=server, + ) + + database_name: Optional[str] = self.get_db_name(data_access_tokens) + + return self.parse_custom_sql( + query=sql_query, + server=server, + database=database_name, + schema=None, + ) + + +class SupportedPattern(Enum): + DATABRICKS_QUERY = ( + DatabricksLineage, + FunctionName.DATABRICK_DATA_ACCESS, + ) + + DATABRICKS_MULTI_CLOUD = ( + DatabricksLineage, + FunctionName.DATABRICK_MULTI_CLOUD_DATA_ACCESS, + ) + + POSTGRES_SQL = ( + PostgresLineage, + FunctionName.POSTGRESQL_DATA_ACCESS, + ) + + ORACLE = ( + OracleLineage, + FunctionName.ORACLE_DATA_ACCESS, + ) + + SNOWFLAKE = ( + SnowflakeLineage, + FunctionName.SNOWFLAKE_DATA_ACCESS, + ) + + MS_SQL = ( + MSSqlLineage, + FunctionName.MSSQL_DATA_ACCESS, + ) + + GOOGLE_BIG_QUERY = ( + GoogleBigQueryLineage, + FunctionName.GOOGLE_BIGQUERY_DATA_ACCESS, + ) + + AMAZON_REDSHIFT = ( + AmazonRedshiftLineage, + FunctionName.AMAZON_REDSHIFT_DATA_ACCESS, + ) + + NATIVE_QUERY = ( + NativeQueryLineage, + FunctionName.NATIVE_QUERY, + ) + + def handler(self) -> Type[AbstractLineage]: + return self.value[0] + + def function_name(self) -> str: + return self.value[1].value + + @staticmethod + def get_function_names() -> List[str]: + functions: List[str] = [] + for supported_resolver in SupportedPattern: + functions.append(supported_resolver.function_name()) + + return functions + + @staticmethod + def get_pattern_handler(function_name: str) -> Optional["SupportedPattern"]: + logger.debug(f"Looking for pattern-handler for {function_name}") + for supported_resolver in SupportedPattern: + if function_name == supported_resolver.function_name(): + return supported_resolver + logger.debug(f"pattern-handler not found for function_name {function_name}") + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py index a40e67d08da5b2..81a0e1ef2d79b1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py @@ -1,286 +1,33 @@ import logging from abc import ABC, abstractmethod -from dataclasses import dataclass -from enum import Enum -from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast +from typing import Any, Dict, List, Optional, Tuple, Union, cast from lark import Tree -import datahub.emitter.mce_builder as builder from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.powerbi.config import ( - Constant, - DataBricksPlatformDetail, - DataPlatformPair, - PlatformDetail, PowerBiDashboardSourceConfig, PowerBiDashboardSourceReport, - PowerBIPlatformDetail, - SupportedDataPlatform, ) from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import ( AbstractDataPlatformInstanceResolver, ) -from datahub.ingestion.source.powerbi.m_query import native_sql_parser, tree_function +from datahub.ingestion.source.powerbi.m_query import tree_function from datahub.ingestion.source.powerbi.m_query.data_classes import ( TRACE_POWERBI_MQUERY_PARSER, - AbstractIdentifierAccessor, DataAccessFunctionDetail, IdentifierAccessor, - ReferencedTable, + Lineage, +) +from datahub.ingestion.source.powerbi.m_query.pattern_handler import ( + AbstractLineage, + SupportedPattern, ) from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table -from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo, SqlParsingResult logger = logging.getLogger(__name__) -@dataclass -class DataPlatformTable: - data_platform_pair: DataPlatformPair - urn: str - - -@dataclass -class Lineage: - upstreams: List[DataPlatformTable] - column_lineage: List[ColumnLineageInfo] - - @staticmethod - def empty() -> "Lineage": - return Lineage(upstreams=[], column_lineage=[]) - - -def urn_to_lowercase(value: str, flag: bool) -> str: - if flag is True: - return value.lower() - - return value - - -def urn_creator( - config: PowerBiDashboardSourceConfig, - platform_instance_resolver: AbstractDataPlatformInstanceResolver, - data_platform_pair: DataPlatformPair, - server: str, - qualified_table_name: str, -) -> str: - platform_detail: PlatformDetail = platform_instance_resolver.get_platform_instance( - PowerBIPlatformDetail( - data_platform_pair=data_platform_pair, - data_platform_server=server, - ) - ) - - return builder.make_dataset_urn_with_platform_instance( - platform=data_platform_pair.datahub_data_platform_name, - platform_instance=platform_detail.platform_instance, - env=platform_detail.env, - name=urn_to_lowercase( - qualified_table_name, config.convert_lineage_urns_to_lowercase - ), - ) - - -def get_next_item(items: List[str], item: str) -> Optional[str]: - if item in items: - try: - index = items.index(item) - return items[index + 1] - except IndexError: - logger.debug(f'item:"{item}", not found in item-list: {items}') - return None - - -class AbstractDataPlatformTableCreator(ABC): - """ - Base class to share common functionalities among different dataplatform for M-Query parsing. - - To create qualified table name we need to parse M-Query data-access-functions(https://learn.microsoft.com/en-us/powerquery-m/accessing-data-functions) and - the data-access-functions has some define pattern to access database-name, schema-name and table-name, for example see below M-Query. - - let - Source = Sql.Database("localhost", "library"), - dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data] - in - dbo_book_issue - - It is MSSQL M-Query and Sql.Database is the data-access-function to access MSSQL. If this function is available in M-Query then database name is available in second argument - of first statement and schema-name and table-name is available in second statement. second statement can be repeated to access different tables from MSSQL. - - DefaultTwoStepDataAccessSources extends the AbstractDataPlatformTableCreator and provides the common functionalities for data-platform which has above type of M-Query pattern - - data-access-function varies as per data-platform for example for MySQL.Database for MySQL, PostgreSQL.Database for Postgres and Oracle.Database for Oracle and number of statement to - find out database-name , schema-name and table-name also varies as per dataplatform. - - Value.NativeQuery is one of the function which is used to execute native query inside M-Query, for example see below M-Query - - let - Source = Value.NativeQuery(AmazonRedshift.Database("redshift-url","dev"), "select * from dev.public.category", null, [EnableFolding=true]) - in - Source - - In this M-Query database-name is available in first argument and rest of the detail i.e database & schema is available in native query. - - NativeQueryDataPlatformTableCreator extends AbstractDataPlatformTableCreator to support Redshift and Snowflake native query parsing. - - """ - - ctx: PipelineContext - table: Table - config: PowerBiDashboardSourceConfig - reporter: PowerBiDashboardSourceReport - platform_instance_resolver: AbstractDataPlatformInstanceResolver - - def __init__( - self, - ctx: PipelineContext, - table: Table, - config: PowerBiDashboardSourceConfig, - reporter: PowerBiDashboardSourceReport, - platform_instance_resolver: AbstractDataPlatformInstanceResolver, - ) -> None: - super().__init__() - self.ctx = ctx - self.table = table - self.config = config - self.reporter = reporter - self.platform_instance_resolver = platform_instance_resolver - - @abstractmethod - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - pass - - @abstractmethod - def get_platform_pair(self) -> DataPlatformPair: - pass - - @staticmethod - def get_db_detail_from_argument( - arg_list: Tree, - ) -> Tuple[Optional[str], Optional[str]]: - arguments: List[str] = tree_function.strip_char_from_list( - values=tree_function.remove_whitespaces_from_list( - tree_function.token_values(arg_list) - ), - ) - - if len(arguments) < 2: - logger.debug(f"Expected minimum 2 arguments, but got {len(arguments)}") - return None, None - - return arguments[0], arguments[1] - - @staticmethod - def create_reference_table( - arg_list: Tree, - table_detail: Dict[str, str], - ) -> Optional[ReferencedTable]: - arguments: List[str] = tree_function.strip_char_from_list( - values=tree_function.remove_whitespaces_from_list( - tree_function.token_values(arg_list) - ), - ) - - logger.debug(f"Processing arguments {arguments}") - - if ( - len(arguments) - >= 4 # [0] is warehouse FQDN. - # [1] is endpoint, we are not using it. - # [2] is "Catalog" key - # [3] is catalog's value - ): - return ReferencedTable( - warehouse=arguments[0], - catalog=arguments[3], - # As per my observation, database and catalog names are same in M-Query - database=table_detail["Database"] - if table_detail.get("Database") - else arguments[3], - schema=table_detail["Schema"], - table=table_detail.get("Table") or table_detail["View"], - ) - elif len(arguments) == 2: - return ReferencedTable( - warehouse=arguments[0], - database=table_detail["Database"], - schema=table_detail["Schema"], - table=table_detail.get("Table") or table_detail["View"], - catalog=None, - ) - - return None - - def parse_custom_sql( - self, query: str, server: str, database: Optional[str], schema: Optional[str] - ) -> Lineage: - dataplatform_tables: List[DataPlatformTable] = [] - - platform_detail: PlatformDetail = ( - self.platform_instance_resolver.get_platform_instance( - PowerBIPlatformDetail( - data_platform_pair=self.get_platform_pair(), - data_platform_server=server, - ) - ) - ) - - query = native_sql_parser.remove_drop_statement( - native_sql_parser.remove_special_characters(query) - ) - - parsed_result: Optional[ - "SqlParsingResult" - ] = native_sql_parser.parse_custom_sql( - ctx=self.ctx, - query=query, - platform=self.get_platform_pair().datahub_data_platform_name, - platform_instance=platform_detail.platform_instance, - env=platform_detail.env, - database=database, - schema=schema, - ) - - if parsed_result is None: - self.reporter.info( - title=Constant.SQL_PARSING_FAILURE, - message="Fail to parse native sql present in PowerBI M-Query", - context=f"table-name={self.table.full_name}, sql={query}", - ) - return Lineage.empty() - - if parsed_result.debug_info and parsed_result.debug_info.table_error: - self.reporter.warning( - title=Constant.SQL_PARSING_FAILURE, - message="Fail to parse native sql present in PowerBI M-Query", - context=f"table-name={self.table.full_name}, error={parsed_result.debug_info.table_error},sql={query}", - ) - return Lineage.empty() - - for urn in parsed_result.in_tables: - dataplatform_tables.append( - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ) - - logger.debug(f"Native Query parsed result={parsed_result}") - logger.debug(f"Generated dataplatform_tables={dataplatform_tables}") - - return Lineage( - upstreams=dataplatform_tables, - column_lineage=( - parsed_result.column_lineage - if parsed_result.column_lineage is not None - else [] - ), - ) - - class AbstractDataAccessMQueryResolver(ABC): table: Table parse_tree: Tree @@ -299,10 +46,10 @@ def __init__( self.parse_tree = parse_tree self.reporter = reporter self.parameters = parameters - self.data_access_functions = SupportedResolver.get_function_names() + self.data_access_functions = SupportedPattern.get_function_names() @abstractmethod - def resolve_to_data_platform_table_list( + def resolve_to_lineage( self, ctx: PipelineContext, config: PowerBiDashboardSourceConfig, @@ -318,7 +65,7 @@ class MQueryResolver(AbstractDataAccessMQueryResolver, ABC): This class has generic code to process M-Query tokens and create instance of DataAccessFunctionDetail. Once DataAccessFunctionDetail instance is initialized thereafter MQueryResolver generates the DataPlatformTable with the help of AbstractDataPlatformTableCreator - (see method resolve_to_data_platform_table_list). + (see method resolve_to_lineage). Classes which extended from AbstractDataPlatformTableCreator know how to convert generated DataAccessFunctionDetail instance to the respective DataPlatformTable instance as per dataplatform. @@ -602,7 +349,7 @@ def internal( return table_links - def resolve_to_data_platform_table_list( + def resolve_to_lineage( self, ctx: PipelineContext, config: PowerBiDashboardSourceConfig, @@ -630,7 +377,7 @@ def resolve_to_data_platform_table_list( # Each item is data-access function for f_detail in table_links: # Get & Check if we support data-access-function available in M-Query - supported_resolver = SupportedResolver.get_resolver( + supported_resolver = SupportedPattern.get_pattern_handler( f_detail.data_access_function_name ) if supported_resolver is None: @@ -643,11 +390,9 @@ def resolve_to_data_platform_table_list( ) continue - # From supported_resolver enum get respective resolver like AmazonRedshift or Snowflake or Oracle or NativeQuery and create instance of it - # & also pass additional information that will be need to generate urn - table_qualified_name_creator: ( - AbstractDataPlatformTableCreator - ) = supported_resolver.get_table_full_name_creator()( + # From supported_resolver enum get respective handler like AmazonRedshift or Snowflake or Oracle or NativeQuery and create instance of it + # & also pass additional information that will be need to generate lineage + pattern_handler: (AbstractLineage) = supported_resolver.handler()( ctx=ctx, table=self.table, config=config, @@ -655,673 +400,6 @@ def resolve_to_data_platform_table_list( platform_instance_resolver=platform_instance_resolver, ) - lineage.append(table_qualified_name_creator.create_lineage(f_detail)) + lineage.append(pattern_handler.create_lineage(f_detail)) return lineage - - -class DefaultTwoStepDataAccessSources(AbstractDataPlatformTableCreator, ABC): - """ - These are the DataSource for which PowerBI Desktop generates default M-Query of following pattern - let - Source = Sql.Database("localhost", "library"), - dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data] - in - dbo_book_issue - """ - - def two_level_access_pattern( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - logger.debug( - f"Processing {self.get_platform_pair().powerbi_data_platform_name} data-access function detail {data_access_func_detail}" - ) - - server, db_name = self.get_db_detail_from_argument( - data_access_func_detail.arg_list - ) - if server is None or db_name is None: - return Lineage.empty() # Return an empty list - - schema_name: str = cast( - IdentifierAccessor, data_access_func_detail.identifier_accessor - ).items["Schema"] - - table_name: str = cast( - IdentifierAccessor, data_access_func_detail.identifier_accessor - ).items["Item"] - - qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" - - logger.debug( - f"Platform({self.get_platform_pair().datahub_data_platform_name}) qualified_table_name= {qualified_table_name}" - ) - - urn = urn_creator( - config=self.config, - platform_instance_resolver=self.platform_instance_resolver, - data_platform_pair=self.get_platform_pair(), - server=server, - qualified_table_name=qualified_table_name, - ) - return Lineage( - upstreams=[ - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ], - column_lineage=[], - ) - - -class PostgresDataPlatformTableCreator(DefaultTwoStepDataAccessSources): - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - return self.two_level_access_pattern(data_access_func_detail) - - def get_platform_pair(self) -> DataPlatformPair: - return SupportedDataPlatform.POSTGRES_SQL.value - - -class MSSqlDataPlatformTableCreator(DefaultTwoStepDataAccessSources): - # https://learn.microsoft.com/en-us/sql/relational-databases/security/authentication-access/ownership-and-user-schema-separation?view=sql-server-ver16 - DEFAULT_SCHEMA = "dbo" # Default schema name in MS-SQL is dbo - - def get_platform_pair(self) -> DataPlatformPair: - return SupportedDataPlatform.MS_SQL.value - - def create_urn_using_old_parser( - self, query: str, db_name: str, server: str - ) -> List[DataPlatformTable]: - dataplatform_tables: List[DataPlatformTable] = [] - - tables: List[str] = native_sql_parser.get_tables(query) - - for parsed_table in tables: - # components: List[str] = [v.strip("[]") for v in parsed_table.split(".")] - components = [v.strip("[]") for v in parsed_table.split(".")] - if len(components) == 3: - database, schema, table = components - elif len(components) == 2: - schema, table = components - database = db_name - elif len(components) == 1: - (table,) = components - database = db_name - schema = MSSqlDataPlatformTableCreator.DEFAULT_SCHEMA - else: - self.reporter.warning( - title="Invalid table format", - message="The advanced SQL lineage feature (enable_advance_lineage_sql_construct) is disabled. Please either enable this feature or ensure the table is referenced as .. in the SQL.", - context=f"table-name={self.table.full_name}", - ) - continue - - qualified_table_name = f"{database}.{schema}.{table}" - urn = urn_creator( - config=self.config, - platform_instance_resolver=self.platform_instance_resolver, - data_platform_pair=self.get_platform_pair(), - server=server, - qualified_table_name=qualified_table_name, - ) - dataplatform_tables.append( - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ) - - logger.debug(f"Generated upstream tables = {dataplatform_tables}") - - return dataplatform_tables - - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - arguments: List[str] = tree_function.strip_char_from_list( - values=tree_function.remove_whitespaces_from_list( - tree_function.token_values(data_access_func_detail.arg_list) - ), - ) - - server, database = self.get_db_detail_from_argument( - data_access_func_detail.arg_list - ) - if server is None or database is None: - return Lineage.empty() # Return an empty list - - assert server - assert database # to silent the lint - - query: Optional[str] = get_next_item(arguments, "Query") - if query: - if self.config.enable_advance_lineage_sql_construct is False: - # Use previous parser to generate URN to keep backward compatibility - return Lineage( - upstreams=self.create_urn_using_old_parser( - query=query, - db_name=database, - server=server, - ), - column_lineage=[], - ) - - return self.parse_custom_sql( - query=query, - database=database, - server=server, - schema=MSSqlDataPlatformTableCreator.DEFAULT_SCHEMA, - ) - - # It is a regular case of MS-SQL - logger.debug("Handling with regular case") - return self.two_level_access_pattern(data_access_func_detail) - - -class OracleDataPlatformTableCreator(AbstractDataPlatformTableCreator): - def get_platform_pair(self) -> DataPlatformPair: - return SupportedDataPlatform.ORACLE.value - - @staticmethod - def _get_server_and_db_name(value: str) -> Tuple[Optional[str], Optional[str]]: - error_message: str = ( - f"The target argument ({value}) should in the format of :/[" - ".]" - ) - splitter_result: List[str] = value.split("/") - if len(splitter_result) != 2: - logger.debug(error_message) - return None, None - - db_name = splitter_result[1].split(".")[0] - - return tree_function.strip_char_from_list([splitter_result[0]])[0], db_name - - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - logger.debug( - f"Processing Oracle data-access function detail {data_access_func_detail}" - ) - - arguments: List[str] = tree_function.remove_whitespaces_from_list( - tree_function.token_values(data_access_func_detail.arg_list) - ) - - server, db_name = self._get_server_and_db_name(arguments[0]) - - if db_name is None or server is None: - return Lineage.empty() - - schema_name: str = cast( - IdentifierAccessor, data_access_func_detail.identifier_accessor - ).items["Schema"] - - table_name: str = cast( - IdentifierAccessor, - cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next, - ).items["Name"] - - qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" - - urn = urn_creator( - config=self.config, - platform_instance_resolver=self.platform_instance_resolver, - data_platform_pair=self.get_platform_pair(), - server=server, - qualified_table_name=qualified_table_name, - ) - - return Lineage( - upstreams=[ - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ], - column_lineage=[], - ) - - -class DatabrickDataPlatformTableCreator(AbstractDataPlatformTableCreator): - def form_qualified_table_name( - self, - table_reference: ReferencedTable, - data_platform_pair: DataPlatformPair, - ) -> str: - platform_detail: PlatformDetail = ( - self.platform_instance_resolver.get_platform_instance( - PowerBIPlatformDetail( - data_platform_pair=data_platform_pair, - data_platform_server=table_reference.warehouse, - ) - ) - ) - - metastore: Optional[str] = None - - qualified_table_name: str = f"{table_reference.database}.{table_reference.schema}.{table_reference.table}" - - if isinstance(platform_detail, DataBricksPlatformDetail): - metastore = platform_detail.metastore - - if metastore is not None: - return f"{metastore}.{qualified_table_name}" - - return qualified_table_name - - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - logger.debug( - f"Processing Databrick data-access function detail {data_access_func_detail}" - ) - table_detail: Dict[str, str] = {} - temp_accessor: Optional[ - Union[IdentifierAccessor, AbstractIdentifierAccessor] - ] = data_access_func_detail.identifier_accessor - - while temp_accessor: - if isinstance(temp_accessor, IdentifierAccessor): - # Condition to handle databricks M-query pattern where table, schema and database all are present in - # the same invoke statement - if all( - element in temp_accessor.items - for element in ["Item", "Schema", "Catalog"] - ): - table_detail["Schema"] = temp_accessor.items["Schema"] - table_detail["Table"] = temp_accessor.items["Item"] - else: - table_detail[temp_accessor.items["Kind"]] = temp_accessor.items[ - "Name" - ] - - if temp_accessor.next is not None: - temp_accessor = temp_accessor.next - else: - break - else: - logger.debug( - "expecting instance to be IdentifierAccessor, please check if parsing is done properly" - ) - return Lineage.empty() - - table_reference = self.create_reference_table( - arg_list=data_access_func_detail.arg_list, - table_detail=table_detail, - ) - - if table_reference: - qualified_table_name: str = self.form_qualified_table_name( - table_reference=table_reference, - data_platform_pair=self.get_platform_pair(), - ) - - urn = urn_creator( - config=self.config, - platform_instance_resolver=self.platform_instance_resolver, - data_platform_pair=self.get_platform_pair(), - server=table_reference.warehouse, - qualified_table_name=qualified_table_name, - ) - - return Lineage( - upstreams=[ - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ], - column_lineage=[], - ) - - return Lineage.empty() - - def get_platform_pair(self) -> DataPlatformPair: - return SupportedDataPlatform.DATABRICK_SQL.value - - -class DefaultThreeStepDataAccessSources(AbstractDataPlatformTableCreator, ABC): - def get_datasource_server( - self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail - ) -> str: - return tree_function.strip_char_from_list([arguments[0]])[0] - - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - logger.debug( - f"Processing {self.get_platform_pair().datahub_data_platform_name} function detail {data_access_func_detail}" - ) - - arguments: List[str] = tree_function.remove_whitespaces_from_list( - tree_function.token_values(data_access_func_detail.arg_list) - ) - # First is database name - db_name: str = data_access_func_detail.identifier_accessor.items["Name"] # type: ignore - # Second is schema name - schema_name: str = cast( - IdentifierAccessor, data_access_func_detail.identifier_accessor.next # type: ignore - ).items["Name"] - # Third is table name - table_name: str = cast( - IdentifierAccessor, data_access_func_detail.identifier_accessor.next.next # type: ignore - ).items["Name"] - - qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" - - logger.debug( - f"{self.get_platform_pair().datahub_data_platform_name} qualified_table_name {qualified_table_name}" - ) - - server: str = self.get_datasource_server(arguments, data_access_func_detail) - - urn = urn_creator( - config=self.config, - platform_instance_resolver=self.platform_instance_resolver, - data_platform_pair=self.get_platform_pair(), - server=server, - qualified_table_name=qualified_table_name, - ) - - return Lineage( - upstreams=[ - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ], - column_lineage=[], - ) - - -class SnowflakeDataPlatformTableCreator(DefaultThreeStepDataAccessSources): - def get_platform_pair(self) -> DataPlatformPair: - return SupportedDataPlatform.SNOWFLAKE.value - - -class GoogleBigQueryDataPlatformTableCreator(DefaultThreeStepDataAccessSources): - def get_platform_pair(self) -> DataPlatformPair: - return SupportedDataPlatform.GOOGLE_BIGQUERY.value - - def get_datasource_server( - self, arguments: List[str], data_access_func_detail: DataAccessFunctionDetail - ) -> str: - # In Google BigQuery server is project-name - # condition to silent lint, it is not going to be None - return ( - data_access_func_detail.identifier_accessor.items["Name"] - if data_access_func_detail.identifier_accessor is not None - else "" - ) - - -class AmazonRedshiftDataPlatformTableCreator(AbstractDataPlatformTableCreator): - def get_platform_pair(self) -> DataPlatformPair: - return SupportedDataPlatform.AMAZON_REDSHIFT.value - - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - logger.debug( - f"Processing AmazonRedshift data-access function detail {data_access_func_detail}" - ) - - server, db_name = self.get_db_detail_from_argument( - data_access_func_detail.arg_list - ) - if db_name is None or server is None: - return Lineage.empty() # Return empty list - - schema_name: str = cast( - IdentifierAccessor, data_access_func_detail.identifier_accessor - ).items["Name"] - - table_name: str = cast( - IdentifierAccessor, - cast(IdentifierAccessor, data_access_func_detail.identifier_accessor).next, - ).items["Name"] - - qualified_table_name: str = f"{db_name}.{schema_name}.{table_name}" - - urn = urn_creator( - config=self.config, - platform_instance_resolver=self.platform_instance_resolver, - data_platform_pair=self.get_platform_pair(), - server=server, - qualified_table_name=qualified_table_name, - ) - - return Lineage( - upstreams=[ - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ], - column_lineage=[], - ) - - -class NativeQueryDataPlatformTableCreator(AbstractDataPlatformTableCreator): - SUPPORTED_NATIVE_QUERY_DATA_PLATFORM: dict = { - SupportedDataPlatform.SNOWFLAKE.value.powerbi_data_platform_name: SupportedDataPlatform.SNOWFLAKE, - SupportedDataPlatform.AMAZON_REDSHIFT.value.powerbi_data_platform_name: SupportedDataPlatform.AMAZON_REDSHIFT, - SupportedDataPlatform.DatabricksMultiCloud_SQL.value.powerbi_data_platform_name: SupportedDataPlatform.DatabricksMultiCloud_SQL, - } - current_data_platform: SupportedDataPlatform = SupportedDataPlatform.SNOWFLAKE - - def get_platform_pair(self) -> DataPlatformPair: - return self.current_data_platform.value - - @staticmethod - def is_native_parsing_supported(data_access_function_name: str) -> bool: - return ( - data_access_function_name - in NativeQueryDataPlatformTableCreator.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM - ) - - def create_urn_using_old_parser(self, query: str, server: str) -> Lineage: - dataplatform_tables: List[DataPlatformTable] = [] - - tables: List[str] = native_sql_parser.get_tables(query) - - for qualified_table_name in tables: - if len(qualified_table_name.split(".")) != 3: - logger.debug( - f"Skipping table {qualified_table_name} as it is not as per qualified_table_name format" - ) - continue - - urn = urn_creator( - config=self.config, - platform_instance_resolver=self.platform_instance_resolver, - data_platform_pair=self.get_platform_pair(), - server=server, - qualified_table_name=qualified_table_name, - ) - - dataplatform_tables.append( - DataPlatformTable( - data_platform_pair=self.get_platform_pair(), - urn=urn, - ) - ) - - logger.debug(f"Generated dataplatform_tables {dataplatform_tables}") - - return Lineage( - upstreams=dataplatform_tables, - column_lineage=[], - ) - - def get_db_name(self, data_access_tokens: List[str]) -> Optional[str]: - if ( - data_access_tokens[0] - != SupportedDataPlatform.DatabricksMultiCloud_SQL.value.powerbi_data_platform_name - ): - return None - - database: Optional[str] = get_next_item(data_access_tokens, "Database") - - if ( - database and database != Constant.M_QUERY_NULL - ): # database name is explicitly set - return database - - return get_next_item( # database name is set in Name argument - data_access_tokens, "Name" - ) or get_next_item( # If both above arguments are not available, then try Catalog - data_access_tokens, "Catalog" - ) - - def create_lineage( - self, data_access_func_detail: DataAccessFunctionDetail - ) -> Lineage: - t1: Tree = cast( - Tree, tree_function.first_arg_list_func(data_access_func_detail.arg_list) - ) - flat_argument_list: List[Tree] = tree_function.flat_argument_list(t1) - - if len(flat_argument_list) != 2: - logger.debug( - f"Expecting 2 argument, actual argument count is {len(flat_argument_list)}" - ) - logger.debug(f"Flat argument list = {flat_argument_list}") - return Lineage.empty() - - data_access_tokens: List[str] = tree_function.remove_whitespaces_from_list( - tree_function.token_values(flat_argument_list[0]) - ) - - if not self.is_native_parsing_supported(data_access_tokens[0]): - logger.debug( - f"Unsupported native-query data-platform = {data_access_tokens[0]}" - ) - logger.debug( - f"NativeQuery is supported only for {self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM}" - ) - - return Lineage.empty() - - if len(data_access_tokens[0]) < 3: - logger.debug( - f"Server is not available in argument list for data-platform {data_access_tokens[0]}. Returning empty " - "list" - ) - return Lineage.empty() - - self.current_data_platform = self.SUPPORTED_NATIVE_QUERY_DATA_PLATFORM[ - data_access_tokens[0] - ] - # The First argument is the query - sql_query: str = tree_function.strip_char_from_list( - values=tree_function.remove_whitespaces_from_list( - tree_function.token_values(flat_argument_list[1]) - ), - )[ - 0 - ] # Remove any whitespaces and double quotes character - - server = tree_function.strip_char_from_list([data_access_tokens[2]])[0] - - if self.config.enable_advance_lineage_sql_construct is False: - # Use previous parser to generate URN to keep backward compatibility - return self.create_urn_using_old_parser( - query=sql_query, - server=server, - ) - - database_name: Optional[str] = self.get_db_name(data_access_tokens) - - return self.parse_custom_sql( - query=sql_query, - server=server, - database=database_name, - schema=None, - ) - - -class FunctionName(Enum): - NATIVE_QUERY = "Value.NativeQuery" - POSTGRESQL_DATA_ACCESS = "PostgreSQL.Database" - ORACLE_DATA_ACCESS = "Oracle.Database" - SNOWFLAKE_DATA_ACCESS = "Snowflake.Databases" - MSSQL_DATA_ACCESS = "Sql.Database" - DATABRICK_DATA_ACCESS = "Databricks.Catalogs" - GOOGLE_BIGQUERY_DATA_ACCESS = "GoogleBigQuery.Database" - AMAZON_REDSHIFT_DATA_ACCESS = "AmazonRedshift.Database" - DATABRICK_MULTI_CLOUD_DATA_ACCESS = "DatabricksMultiCloud.Catalogs" - - -class SupportedResolver(Enum): - DATABRICKS_QUERY = ( - DatabrickDataPlatformTableCreator, - FunctionName.DATABRICK_DATA_ACCESS, - ) - - DATABRICKS_MULTI_CLOUD = ( - DatabrickDataPlatformTableCreator, - FunctionName.DATABRICK_MULTI_CLOUD_DATA_ACCESS, - ) - - POSTGRES_SQL = ( - PostgresDataPlatformTableCreator, - FunctionName.POSTGRESQL_DATA_ACCESS, - ) - - ORACLE = ( - OracleDataPlatformTableCreator, - FunctionName.ORACLE_DATA_ACCESS, - ) - - SNOWFLAKE = ( - SnowflakeDataPlatformTableCreator, - FunctionName.SNOWFLAKE_DATA_ACCESS, - ) - - MS_SQL = ( - MSSqlDataPlatformTableCreator, - FunctionName.MSSQL_DATA_ACCESS, - ) - - GOOGLE_BIG_QUERY = ( - GoogleBigQueryDataPlatformTableCreator, - FunctionName.GOOGLE_BIGQUERY_DATA_ACCESS, - ) - - AMAZON_REDSHIFT = ( - AmazonRedshiftDataPlatformTableCreator, - FunctionName.AMAZON_REDSHIFT_DATA_ACCESS, - ) - - NATIVE_QUERY = ( - NativeQueryDataPlatformTableCreator, - FunctionName.NATIVE_QUERY, - ) - - def get_table_full_name_creator(self) -> Type[AbstractDataPlatformTableCreator]: - return self.value[0] - - def get_function_name(self) -> str: - return self.value[1].value - - @staticmethod - def get_function_names() -> List[str]: - functions: List[str] = [] - for supported_resolver in SupportedResolver: - functions.append(supported_resolver.get_function_name()) - - return functions - - @staticmethod - def get_resolver(function_name: str) -> Optional["SupportedResolver"]: - logger.debug(f"Looking for resolver {function_name}") - for supported_resolver in SupportedResolver: - if function_name == supported_resolver.get_function_name(): - return supported_resolver - logger.debug(f"Resolver not found for function_name {function_name}") - return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py index ca2abf97c9f303..b52977aaa41fbe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/validator.py @@ -1,7 +1,7 @@ import logging from typing import Optional, Tuple -from datahub.ingestion.source.powerbi.m_query import resolver +import datahub.ingestion.source.powerbi.m_query.data_classes logger = logging.getLogger(__name__) @@ -14,12 +14,18 @@ def validate_parse_tree( :param native_query_enabled: Whether user want to extract lineage from native query :return: True or False. """ - function_names = [fun.value for fun in resolver.FunctionName] + function_names = [ + fun.value + for fun in datahub.ingestion.source.powerbi.m_query.data_classes.FunctionName + ] if not any(fun in expression for fun in function_names): return False, "DataAccess function is not present in M-Query expression." if native_query_enabled is False: - if resolver.FunctionName.NATIVE_QUERY.value in function_names: + if ( + datahub.ingestion.source.powerbi.m_query.data_classes.FunctionName.NATIVE_QUERY.value + in function_names + ): return ( False, "Lineage extraction from native query is disabled. Enable native_query_parsing in recipe", diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index cef2d098aebc40..044946a5d308d1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -10,6 +10,7 @@ import more_itertools import datahub.emitter.mce_builder as builder +import datahub.ingestion.source.powerbi.m_query.data_classes import datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes as powerbi_data_classes from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import ContainerKey, gen_containers @@ -42,12 +43,13 @@ Constant, PowerBiDashboardSourceConfig, PowerBiDashboardSourceReport, + SupportedDataPlatform, ) from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import ( AbstractDataPlatformInstanceResolver, create_dataplatform_instance_resolver, ) -from datahub.ingestion.source.powerbi.m_query import parser, resolver +from datahub.ingestion.source.powerbi.m_query import parser from datahub.ingestion.source.powerbi.rest_api_wrapper.powerbi_api import PowerBiAPI from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, @@ -182,7 +184,9 @@ def extract_dataset_schema( return [schema_mcp] def make_fine_grained_lineage_class( - self, lineage: resolver.Lineage, dataset_urn: str + self, + lineage: datahub.ingestion.source.powerbi.m_query.data_classes.Lineage, + dataset_urn: str, ) -> List[FineGrainedLineage]: fine_grained_lineages: List[FineGrainedLineage] = [] @@ -234,7 +238,9 @@ def extract_lineage( upstream: List[UpstreamClass] = [] cll_lineage: List[FineGrainedLineage] = [] - upstream_lineage: List[resolver.Lineage] = parser.get_upstream_tables( + upstream_lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = parser.get_upstream_tables( table=table, reporter=self.__reporter, platform_instance_resolver=self.__dataplatform_instance_resolver, @@ -1294,7 +1300,7 @@ def get_allowed_workspaces(self) -> List[powerbi_data_classes.Workspace]: def validate_dataset_type_mapping(self): powerbi_data_platforms: List[str] = [ data_platform.value.powerbi_data_platform_name - for data_platform in resolver.SupportedDataPlatform + for data_platform in SupportedDataPlatform ] for key in self.source_config.dataset_type_mapping.keys(): @@ -1481,7 +1487,7 @@ def _get_dashboard_patch_work_unit( def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: # As modified_workspaces is not idempotent, hence workunit processors are run later for each workspace_id - # This will result in creating checkpoint for each workspace_id + # This will result in creating a checkpoint for each workspace_id if self.source_config.modified_since: return [] # Handle these in get_workunits_internal else: @@ -1492,7 +1498,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: """ - Datahub Ingestion framework invoke this method + Datahub Ingestion framework invokes this method """ logger.info("PowerBi plugin execution is started") # Validate dataset type mapping diff --git a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py index f22998b47b9008..63821f9038a88c 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_m_parser.py +++ b/metadata-ingestion/tests/integration/powerbi/test_m_parser.py @@ -7,6 +7,7 @@ import pytest from lark import Tree +import datahub.ingestion.source.powerbi.m_query.data_classes import datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes as powerbi_data_classes from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.source import StructuredLogLevel @@ -18,8 +19,11 @@ AbstractDataPlatformInstanceResolver, create_dataplatform_instance_resolver, ) -from datahub.ingestion.source.powerbi.m_query import parser, resolver, tree_function -from datahub.ingestion.source.powerbi.m_query.resolver import DataPlatformTable, Lineage +from datahub.ingestion.source.powerbi.m_query import parser, tree_function +from datahub.ingestion.source.powerbi.m_query.data_classes import ( + DataPlatformTable, + Lineage, +) pytestmark = pytest.mark.integration_batch_2 @@ -62,7 +66,9 @@ ] -def get_data_platform_tables_with_dummy_table(q: str) -> List[resolver.Lineage]: +def get_data_platform_tables_with_dummy_table( + q: str, +) -> List[datahub.ingestion.source.powerbi.m_query.data_classes.Lineage]: table: powerbi_data_classes.Table = powerbi_data_classes.Table( columns=[], measures=[], @@ -759,7 +765,9 @@ def test_sqlglot_parser(): } ) - lineage: List[resolver.Lineage] = parser.get_upstream_tables( + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = parser.get_upstream_tables( table, reporter, ctx=ctx, @@ -806,7 +814,9 @@ def test_sqlglot_parser(): def test_databricks_multi_cloud(): q = M_QUERIES[25] - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 @@ -823,7 +833,9 @@ def test_databricks_multi_cloud(): def test_databricks_catalog_pattern_1(): q = M_QUERIES[26] - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 @@ -892,7 +904,9 @@ def test_sqlglot_parser_2(): } ) - lineage: List[resolver.Lineage] = parser.get_upstream_tables( + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = parser.get_upstream_tables( table, reporter, ctx=ctx, @@ -951,7 +965,9 @@ def test_databricks_regular_case_with_view(): def test_snowflake_double_double_quotes(): q = M_QUERIES[30] - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 @@ -968,7 +984,9 @@ def test_snowflake_double_double_quotes(): def test_databricks_multicloud(): q = M_QUERIES[31] - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 @@ -985,7 +1003,9 @@ def test_databricks_multicloud(): def test_snowflake_multi_function_call(): q = M_QUERIES[32] - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 @@ -1002,7 +1022,9 @@ def test_snowflake_multi_function_call(): def test_mssql_drop_with_select(): q = M_QUERIES[33] - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 @@ -1062,7 +1084,9 @@ def test_empty_string_in_m_query(): # TRIM(TRIM(TRIM(AGENT_NAME, '\"\"'), '+'), '\\'') is in Query q = "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu10758.ap-unknown-2.fakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"select #(lf)UPPER(REPLACE(AGENT_NAME,'-','')) AS CLIENT_DIRECTOR,#(lf)TRIM(TRIM(TRIM(AGENT_NAME, '\"\"'), '+'), '\\'') AS TRIM_AGENT_NAME,#(lf)TIER,#(lf)UPPER(MANAGER),#(lf)TEAM_TYPE,#(lf)DATE_TARGET,#(lf)MONTHID,#(lf)TARGET_TEAM,#(lf)SELLER_EMAIL,#(lf)concat((UPPER(REPLACE(AGENT_NAME,'-',''))), MONTHID) as AGENT_KEY,#(lf)UNIT_TARGET AS SME_Quota,#(lf)AMV_TARGET AS Revenue_Quota,#(lf)SERVICE_QUOTA,#(lf)BL_TARGET,#(lf)SOFTWARE_QUOTA as Software_Quota#(lf)#(lf)from OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_SME_UNIT_TARGETS inner join OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_SME_UNIT #(lf)#(lf)where YEAR_TARGET >= 2022#(lf)and TEAM_TYPE = 'Accounting'#(lf)and TARGET_TEAM = 'Enterprise'#(lf)AND TIER = 'Client Director'\", null, [EnableFolding=true])\nin\n Source" - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 @@ -1084,7 +1108,9 @@ def test_double_quotes_in_alias(): # SELECT CAST(sales_date AS DATE) AS \"\"Date\"\" in query q = 'let \n Source = Sql.Database("abc.com", "DB", [Query="SELECT CAST(sales_date AS DATE) AS ""Date"",#(lf) SUM(cshintrpret) / 60.0 AS ""Total Order All Items"",#(lf)#(tab)#(tab)#(tab) SUM(cshintrpret) / 60.0 - LAG(SUM(cshintrpret) / 60.0, 1) OVER (ORDER BY CAST(sales_date AS DATE)) AS ""Total minute difference"",#(lf)#(tab)#(tab)#(tab) SUM(sale_price) / 60.0 - LAG(SUM(sale_price) / 60.0, 1) OVER (ORDER BY CAST(sales_date AS DATE)) AS ""Normal minute difference""#(lf) FROM [DB].[dbo].[sales_t]#(lf) WHERE sales_date >= GETDATE() - 365#(lf) GROUP BY CAST(sales_date AS DATE),#(lf)#(tab)#(tab)CAST(sales_date AS TIME);"]) \n in \n Source' - lineage: List[resolver.Lineage] = get_data_platform_tables_with_dummy_table(q=q) + lineage: List[ + datahub.ingestion.source.powerbi.m_query.data_classes.Lineage + ] = get_data_platform_tables_with_dummy_table(q=q) assert len(lineage) == 1 diff --git a/metadata-ingestion/tests/unit/test_powerbi_parser.py b/metadata-ingestion/tests/unit/test_powerbi_parser.py index 31579f0c0abd3e..a487a3a5b87f8b 100644 --- a/metadata-ingestion/tests/unit/test_powerbi_parser.py +++ b/metadata-ingestion/tests/unit/test_powerbi_parser.py @@ -8,9 +8,7 @@ from datahub.ingestion.source.powerbi.dataplatform_instance_resolver import ( ResolvePlatformInstanceFromDatasetTypeMapping, ) -from datahub.ingestion.source.powerbi.m_query.resolver import ( - MSSqlDataPlatformTableCreator, -) +from datahub.ingestion.source.powerbi.m_query.pattern_handler import MSSqlLineage from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import Table @@ -27,7 +25,7 @@ def creator(): full_name="db.schema.test_table", ) - return MSSqlDataPlatformTableCreator( + return MSSqlLineage( ctx=PipelineContext(run_id="test-run-id"), table=table, reporter=PowerBiDashboardSourceReport(), From 0ee758c52dd06665c3998a380cc66278d8b4d959 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 27 Nov 2024 12:10:23 -0600 Subject: [PATCH 087/174] fix(schematron): fix for jdk8 (#11975) --- .../converters/avro/AvroSchemaConverter.java | 12 ++++++++---- .../datahubproject/schematron/models/FieldPath.java | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java index c199f8e6dcb92e..0ddb357db76ba1 100644 --- a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverter.java @@ -345,7 +345,8 @@ private void processArrayField( log.debug("Array Field Path before expand: {}", fieldPath.asString()); fieldPath = fieldPath.popLast(); fieldPath = - fieldPath.clonePlus(new FieldElement(List.of("array"), new ArrayList<>(), null, null)); + fieldPath.clonePlus( + new FieldElement(Collections.singletonList("array"), new ArrayList<>(), null, null)); Schema.Field elementField = new Schema.Field( field.name(), @@ -400,7 +401,9 @@ private void processMapField( FieldPath valueFieldPath = fieldPath .popLast() - .clonePlus(new FieldElement(List.of("map"), new ArrayList<>(), null, null)); + .clonePlus( + new FieldElement( + Collections.singletonList("map"), new ArrayList<>(), null, null)); processField(valueField, valueFieldPath, defaultNullable, fields, isNullable, mapDataHubType); } else { SchemaField mapField = @@ -434,7 +437,7 @@ private void processUnionField( unionTypes.stream() .filter(s -> s.getType() != Schema.Type.NULL) .findFirst() - .orElseThrow(); + .orElseThrow(NoSuchElementException::new); processField( new Schema.Field(field.name(), nonNullSchema, field.doc()), @@ -476,7 +479,8 @@ private void processUnionField( FieldPath indexedFieldPath = fieldPath.popLast(); indexedFieldPath = indexedFieldPath.clonePlus( - new FieldElement(List.of("union"), new ArrayList<>(), null, null)); + new FieldElement( + Collections.singletonList("union"), new ArrayList<>(), null, null)); log.debug("TypeIndex: {}, Indexed Field path : {}", typeIndex, indexedFieldPath.asString()); // FieldPath unionFieldPath = // fieldPath.expandType(getDiscriminatedType(unionSchema), diff --git a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java index e51aa1221c54e0..b4b72fcc031a59 100644 --- a/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java +++ b/metadata-integration/java/datahub-schematron/lib/src/main/java/io/datahubproject/schematron/models/FieldPath.java @@ -2,6 +2,7 @@ import com.linkedin.schema.*; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -117,8 +118,8 @@ public FieldPath expandType(String type, Object typeSchema) { .getPath() .add( new FieldElement( - new ArrayList<>(List.of(type)), - new ArrayList<>(List.of(typeSchema.toString())), + new ArrayList<>(Collections.singletonList(type)), + new ArrayList<>(Collections.singletonList(typeSchema.toString())), null, null)); } From 8d92b28bca432ead30da1563554744c3c9c9d377 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 27 Nov 2024 13:24:09 -0800 Subject: [PATCH 088/174] fix(automations docs): Update snowflake-tag-propagation.md to include permissions required for the Automation (#11977) --- docs/automations/snowflake-tag-propagation.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/automations/snowflake-tag-propagation.md b/docs/automations/snowflake-tag-propagation.md index b72224642b0f07..8eded451644cce 100644 --- a/docs/automations/snowflake-tag-propagation.md +++ b/docs/automations/snowflake-tag-propagation.md @@ -4,6 +4,8 @@ import FeatureAvailability from '@site/src/components/FeatureAvailability'; +> Note that this Automation in currently in open **Beta**. With any questions or issues, please reach out to your Acryl representative. + ## Introduction Snowflake Tag Propagation is an automation that allows you to sync DataHub Glossary Terms and Tags on @@ -15,6 +17,41 @@ both columns and tables back to Snowflake. This automation is available in DataH - Automatically Add DataHub Tags to Snowflake Tables and Columns - Automatically Remove DataHub Glossary Terms and Tags from Snowflake Tables and Columns when they are removed in DataHub +## Prerequisites + +### Permissions Required for Tag Management + +- `CREATE TAG`: Required to create new tags in Snowflake. +Ensure the user or role has this privilege on the specific schema or database where tags will be created. +- `APPLY TAG`: Required to assign tags to Snowflake objects such as tables, columns, or other database objects. +This permission must be granted at the database, schema, or object level depending on the scope. + + +### Permissions Required for Object Access + +- `USAGE` on the database and schema: Allows access to the database and schema to view and apply changes. +- `SELECT` on the objects (tables, views, etc.): Enables the automation to read metadata and verify existing tags. + +### Example Permission Grant Statements + +To grant the necessary permissions for a specific role (DATAHUB_AUTOMATION_ROLE), you can use the following SQL commands: + +```sql +-- Tag management permissions +GRANT CREATE TAG ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT APPLY TAG ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; + +-- Object access for metadata operations +GRANT USAGE ON DATABASE your_database TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT USAGE ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT SELECT ON ALL TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; + +-- Future privileges for tagging +GRANT SELECT ON FUTURE TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT APPLY TAG ON FUTURE TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +``` + + ## Enabling Snowflake Tag Sync 1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. From d9d6255373f1ae2ce4638f4cb12c618013bea57e Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 27 Nov 2024 17:05:37 -0600 Subject: [PATCH 089/174] chore(bump): bump version of akka for datahub-frontend (#11979) --- build.gradle | 10 +++++++++- datahub-frontend/play.gradle | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e3c4f5efe6bb63..be4d7ee8a562b9 100644 --- a/build.gradle +++ b/build.gradle @@ -48,6 +48,7 @@ buildscript { // see also datahub-frontend/play.gradle ext.playVersion = '2.8.22' ext.playScalaVersion = '2.13' + ext.akkaVersion = '2.6.21' // 2.7.0+ has incompatible license ext.log4jVersion = '2.23.1' ext.slf4jVersion = '1.7.36' ext.logbackClassic = '1.4.14' @@ -105,7 +106,14 @@ project.ext.spec = [ ] project.ext.externalDependency = [ - 'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", + 'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10", // max version due to licensing + 'akkaActor': "com.typesafe.akka:akka-actor_$playScalaVersion:$akkaVersion", + 'akkaStream': "com.typesafe.akka:akka-stream_$playScalaVersion:$akkaVersion", + 'akkaActorTyped': "com.typesafe.akka:akka-actor-typed_$playScalaVersion:$akkaVersion", + 'akkaSlf4j': "com.typesafe.akka:akka-slf4j_$playScalaVersion:$akkaVersion", + 'akkaJackson': "com.typesafe.akka:akka-serialization-jackson_$playScalaVersion:$akkaVersion", + 'akkaParsing': "com.typesafe.akka:akka-parsing_$playScalaVersion:$akkaVersion", + 'akkaProtobuf': "com.typesafe.akka:akka-protobuf-v3_$playScalaVersion:$akkaVersion", 'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3', 'antlr4': 'org.antlr:antlr4:4.9.3', 'assertJ': 'org.assertj:assertj-core:3.11.1', diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index 266962721a80a8..d513c3c232d9a0 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -55,6 +55,13 @@ dependencies { implementation externalDependency.antlr4Runtime implementation externalDependency.antlr4 implementation externalDependency.akkaHttp + implementation externalDependency.akkaActor + implementation externalDependency.akkaStream + implementation externalDependency.akkaActorTyped + implementation externalDependency.akkaSlf4j + implementation externalDependency.akkaJackson + implementation externalDependency.akkaParsing + implementation externalDependency.akkaProtobuf implementation externalDependency.jerseyCore implementation externalDependency.jerseyGuava From f3eda3132630803ee1094bce6ee6c97e8b789660 Mon Sep 17 00:00:00 2001 From: Margarida Fernandes Date: Wed, 27 Nov 2024 23:41:04 +0000 Subject: [PATCH 090/174] feat(ingestion): extend feast plugin to ingest tags and owners (#11784) Co-authored-by: John Joyce --- .../src/datahub/ingestion/source/feast.py | 103 +++++++++++++- .../feast/feast_repository_mces_golden.json | 129 ++++++++++++++++-- .../feast/feature_store/data/registry.db | Bin 2715 -> 2991 bytes .../feast/feature_store/features.py | 7 +- .../feast/test_feast_repository.py | 9 ++ 5 files changed, 229 insertions(+), 19 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/feast.py b/metadata-ingestion/src/datahub/ingestion/source/feast.py index e097fd1f221ea5..6330fe0291660d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/feast.py +++ b/metadata-ingestion/src/datahub/ingestion/source/feast.py @@ -42,10 +42,14 @@ from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( BrowsePathsClass, + GlobalTagsClass, MLFeaturePropertiesClass, MLFeatureTablePropertiesClass, MLPrimaryKeyPropertiesClass, + OwnerClass, + OwnershipClass, StatusClass, + TagAssociationClass, ) # FIXME: ValueType module cannot be used as a type @@ -91,6 +95,24 @@ class FeastRepositorySourceConfig(ConfigModel): environment: str = Field( default=DEFAULT_ENV, description="Environment to use when constructing URNs" ) + # owner_mappings example: + # This must be added to the recipe in order to extract owners, otherwise NO owners will be extracted + # owner_mappings: + # - feast_owner_name: "" + # datahub_owner_urn: "urn:li:corpGroup:" + # datahub_ownership_type: "BUSINESS_OWNER" + owner_mappings: Optional[List[Dict[str, str]]] = Field( + default=None, description="Mapping of owner names to owner types" + ) + enable_owner_extraction: bool = Field( + default=False, + description="If this is disabled, then we NEVER try to map owners. " + "If this is enabled, then owner_mappings is REQUIRED to extract ownership.", + ) + enable_tag_extraction: bool = Field( + default=False, + description="If this is disabled, then we NEVER try to extract tags.", + ) @platform_name("Feast") @@ -215,10 +237,15 @@ def _get_entity_workunit( """ feature_view_name = f"{self.feature_store.project}.{feature_view.name}" + aspects = ( + [StatusClass(removed=False)] + + self._get_tags(entity) + + self._get_owners(entity) + ) entity_snapshot = MLPrimaryKeySnapshot( urn=builder.make_ml_primary_key_urn(feature_view_name, entity.name), - aspects=[StatusClass(removed=False)], + aspects=aspects, ) entity_snapshot.aspects.append( @@ -243,10 +270,11 @@ def _get_feature_workunit( Generate an MLFeature work unit for a Feast feature. """ feature_view_name = f"{self.feature_store.project}.{feature_view.name}" + aspects = [StatusClass(removed=False)] + self._get_tags(field) feature_snapshot = MLFeatureSnapshot( urn=builder.make_ml_feature_urn(feature_view_name, field.name), - aspects=[StatusClass(removed=False)], + aspects=aspects, ) feature_sources = [] @@ -295,13 +323,18 @@ def _get_feature_view_workunit(self, feature_view: FeatureView) -> MetadataWorkU """ feature_view_name = f"{self.feature_store.project}.{feature_view.name}" + aspects = ( + [ + BrowsePathsClass(paths=[f"/feast/{self.feature_store.project}"]), + StatusClass(removed=False), + ] + + self._get_tags(feature_view) + + self._get_owners(feature_view) + ) feature_view_snapshot = MLFeatureTableSnapshot( urn=builder.make_ml_feature_table_urn("feast", feature_view_name), - aspects=[ - BrowsePathsClass(paths=[f"/feast/{self.feature_store.project}"]), - StatusClass(removed=False), - ], + aspects=aspects, ) feature_view_snapshot.aspects.append( @@ -360,6 +393,64 @@ def _get_on_demand_feature_view_workunit( return MetadataWorkUnit(id=on_demand_feature_view_name, mce=mce) + # If a tag is specified in a Feast object, then the tag will be ingested into Datahub if enable_tag_extraction is + # True, otherwise NO tags will be ingested + def _get_tags(self, obj: Union[Entity, FeatureView, FeastField]) -> list: + """ + Extracts tags from the given object and returns a list of aspects. + """ + aspects: List[Union[GlobalTagsClass]] = [] + + # Extract tags + if self.source_config.enable_tag_extraction: + if obj.tags.get("name"): + tag_name: str = obj.tags["name"] + tag_association = TagAssociationClass( + tag=builder.make_tag_urn(tag_name) + ) + global_tags_aspect = GlobalTagsClass(tags=[tag_association]) + aspects.append(global_tags_aspect) + + return aspects + + # If an owner is specified in a Feast object, it will only be ingested into Datahub if owner_mappings is specified + # and enable_owner_extraction is True in FeastRepositorySourceConfig, otherwise NO owners will be ingested + def _get_owners(self, obj: Union[Entity, FeatureView, FeastField]) -> list: + """ + Extracts owners from the given object and returns a list of aspects. + """ + aspects: List[Union[OwnershipClass]] = [] + + # Extract owner + if self.source_config.enable_owner_extraction: + owner = getattr(obj, "owner", None) + if owner: + # Create owner association, skipping if None + owner_association = self._create_owner_association(owner) + if owner_association: # Only add valid owner associations + owners_aspect = OwnershipClass(owners=[owner_association]) + aspects.append(owners_aspect) + + return aspects + + def _create_owner_association(self, owner: str) -> Optional[OwnerClass]: + """ + Create an OwnerClass instance for the given owner using the owner mappings. + """ + if self.source_config.owner_mappings is not None: + for mapping in self.source_config.owner_mappings: + if mapping["feast_owner_name"] == owner: + ownership_type_class: str = mapping.get( + "datahub_ownership_type", "TECHNICAL_OWNER" + ) + datahub_owner_urn = mapping.get("datahub_owner_urn") + if datahub_owner_urn: + return OwnerClass( + owner=datahub_owner_urn, + type=ownership_type_class, + ) + return None + @classmethod def create(cls, config_dict, ctx): config = FeastRepositorySourceConfig.parse_obj(config_dict) diff --git a/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json b/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json index 1b91925289845b..a4fd9843c5cf49 100644 --- a/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json +++ b/metadata-ingestion/tests/integration/feast/feast_repository_mces_golden.json @@ -9,8 +9,33 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:deprecated" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpGroup:MOCK_OWNER", + "type": "BUSINESS_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, { "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties": { + "customProperties": {}, "description": "Driver ID", "dataType": "ORDINAL", "sources": [ @@ -23,7 +48,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -36,8 +62,18 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:needs_documentation" + } + ] + } + }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, "description": "Conv rate", "dataType": "CONTINUOUS", "sources": [ @@ -50,7 +86,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -65,6 +102,7 @@ }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, "description": "Acc rate", "dataType": "CONTINUOUS", "sources": [ @@ -77,7 +115,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -92,6 +131,7 @@ }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, "description": "Avg daily trips", "dataType": "ORDINAL", "sources": [ @@ -104,7 +144,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -119,6 +160,7 @@ }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, "description": "String feature", "dataType": "TEXT", "sources": [ @@ -131,7 +173,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -151,6 +194,30 @@ "removed": false } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:deprecated" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpGroup:MOCK_OWNER", + "type": "BUSINESS_OWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties": { "customProperties": {}, @@ -170,7 +237,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -189,7 +257,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -204,6 +273,7 @@ }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, "dataType": "CONTINUOUS", "sources": [ "urn:li:dataset:(urn:li:dataPlatform:request,vals_to_add,PROD)", @@ -216,7 +286,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -231,6 +302,7 @@ }, { "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties": { + "customProperties": {}, "dataType": "CONTINUOUS", "sources": [ "urn:li:dataset:(urn:li:dataPlatform:request,vals_to_add,PROD)", @@ -243,7 +315,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -278,7 +351,8 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } }, { @@ -297,7 +371,40 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "feast-repository-test" + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:deprecated", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "deprecated" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:needs_documentation", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "needs_documentation" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "feast-repository-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/feast/feature_store/data/registry.db b/metadata-ingestion/tests/integration/feast/feature_store/data/registry.db index a511ff56c97705ff3e98d4b90b756e67523d6af5..5dca29d92afe53ffd713ad01267c58f844037f48 100644 GIT binary patch delta 813 zcmbO&x?X&OuI&EC>ATnjE*zTnib;rv1Bu5~#1%BrOS#@jh>ImJF*j9+Dq_ikCzPKd6C^d-7*WcMY-ap*WHAqN?i-+URF_57%n!X-k1R9FOlVUVf zQb{s2wM;WlOw~0?Of%6nF*P#PwKPvO)-_EsvM@DDHZ`+MO3~usII({o&=Bs9AAK_! zjlME-U16Raz$`ZLt(3M47iV&QURit*&@BS2QqoWd3g@M!rWD7g>bV3!f0u@f)$B-|xnkyZQNgRh=>ogVMHbl`2${GmWq_ikCzPKd6C{;*? zi-)6kHAvHs!;_CO12rM>q!MRZ^3|FjTy2JrAOoDATO9RVPfyp~q>}9!}n*?L^GxBp&^;46Jlk|&=lJ)Z^ zzvXnA9K)u<<8AHC!NASIB*G}dH2ET%qK~kjU&)l9DYZb@!yg}?n3tCi471GqyyEzH zpqNGme~(~6VqQvOv0gGT?)1`%5_40hWbjU&(!*Ai3XIgeDH)2T#cflPGQ>CAvHLK| zK@z})1nnhw#l6i`uB>?}5(l7u3 diff --git a/metadata-ingestion/tests/integration/feast/feature_store/features.py b/metadata-ingestion/tests/integration/feast/feature_store/features.py index a6e6cd3616e924..dcfd417637958c 100644 --- a/metadata-ingestion/tests/integration/feast/feature_store/features.py +++ b/metadata-ingestion/tests/integration/feast/feature_store/features.py @@ -19,6 +19,8 @@ join_keys=["driver_id"], value_type=ValueType.INT64, description="Driver ID", + owner="MOCK_OWNER", + tags={"name": "deprecated"}, ) driver_hourly_stats_view = FeatureView( @@ -29,7 +31,7 @@ Field( name="conv_rate", dtype=feast.types.Float64, - tags=dict(description="Conv rate"), + tags={"name": "needs_documentation", "description": "Conv rate"}, ), Field( name="acc_rate", @@ -49,7 +51,8 @@ ], online=True, source=driver_hourly_stats_source, - tags={}, + tags={"name": "deprecated"}, + owner="MOCK_OWNER", ) input_request = RequestSource( diff --git a/metadata-ingestion/tests/integration/feast/test_feast_repository.py b/metadata-ingestion/tests/integration/feast/test_feast_repository.py index a6bdce67222896..7f04337145dc36 100644 --- a/metadata-ingestion/tests/integration/feast/test_feast_repository.py +++ b/metadata-ingestion/tests/integration/feast/test_feast_repository.py @@ -19,6 +19,15 @@ def test_feast_repository_ingest(pytestconfig, tmp_path, mock_time): "config": { "path": str(test_resources_dir / "feature_store"), "environment": "PROD", + "enable_tag_extraction": True, + "enable_owner_extraction": True, + "owner_mappings": [ + { + "feast_owner_name": "MOCK_OWNER", + "datahub_owner_urn": "urn:li:corpGroup:MOCK_OWNER", + "datahub_ownership_type": "BUSINESS_OWNER", + } + ], }, }, "sink": { From 9dfcc0c3d19fb477dac5555b8cbcd7544de5ac41 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 27 Nov 2024 20:03:28 -0600 Subject: [PATCH 091/174] fix(validation): additional URN validation adjustments (#11973) --- docs/what/urn.md | 16 +++-- metadata-io/metadata-io-api/build.gradle | 4 ++ .../entity/validation/ValidationApiUtils.java | 24 +++++--- .../validation/ValidationApiUtilsTest.java | 58 ++++++++++++++++--- 4 files changed, 82 insertions(+), 20 deletions(-) diff --git a/docs/what/urn.md b/docs/what/urn.md index 2f4dffb985653c..c7fb0555cd992f 100644 --- a/docs/what/urn.md +++ b/docs/what/urn.md @@ -35,11 +35,17 @@ urn:li:dataset:(urn:li:dataPlatform:hdfs,PageViewEvent,EI) ## Restrictions -There are a few restrictions when creating an urn: +There are a few restrictions when creating an URN: -1. Commas are reserved character in URN fields: `,` -2. Parentheses are reserved characters in URN fields: `(` or `)` -3. Colons are reserved characters in URN fields: `:` -4. Urn separator UTF-8 character `␟` +The following characters are not allowed anywhere in the URN + +1. Parentheses are reserved characters in URN fields: `(` or `)` +2. The "unit separator" unicode character `␟` (U+241F) + +The following characters are not allowed within an URN tuple. + +1. Commas are reserved characters in URN tuples: `,` + +Example: `urn:li:dashboard:(looker,dashboards.thelook)` is a valid urn, but `urn:li:dashboard:(looker,dashboards.the,look)` is invalid. Please do not use these characters when creating or generating urns. One approach is to use URL encoding for the characters. diff --git a/metadata-io/metadata-io-api/build.gradle b/metadata-io/metadata-io-api/build.gradle index b8028fad07bb65..5273177b752819 100644 --- a/metadata-io/metadata-io-api/build.gradle +++ b/metadata-io/metadata-io-api/build.gradle @@ -16,3 +16,7 @@ dependencies { testImplementation externalDependency.lombok testAnnotationProcessor externalDependency.lombok } + +test { + environment 'STRICT_URN_VALIDATION_ENABLED', 'true' +} \ No newline at end of file diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java index c2e1c47eca1fdb..5e1f09fcc64393 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/validation/ValidationApiUtils.java @@ -30,7 +30,8 @@ public class ValidationApiUtils { // Related to BrowsePathv2 public static final String URN_DELIMITER_SEPARATOR = "␟"; // https://datahubproject.io/docs/what/urn/#restrictions - public static final Set ILLEGAL_URN_COMPONENT_CHARACTERS = Set.of(":", "(", ")", ","); + public static final Set ILLEGAL_URN_COMPONENT_CHARACTERS = Set.of("(", ")"); + public static final Set ILLEGAL_URN_TUPLE_CHARACTERS = Set.of(","); /** * Validates a {@link RecordTemplate} and throws {@link ValidationException} if validation fails. @@ -86,11 +87,10 @@ public static void validateUrn( "Error: URN cannot contain " + URN_DELIMITER_SEPARATOR + " character"); } + int totalParts = urn.getEntityKey().getParts().size(); List illegalComponents = urn.getEntityKey().getParts().stream() - .flatMap(ValidationApiUtils::processUrnPartRecursively) - .filter( - urnPart -> ILLEGAL_URN_COMPONENT_CHARACTERS.stream().anyMatch(urnPart::contains)) + .flatMap(part -> processUrnPartRecursively(part, totalParts)) .collect(Collectors.toList()); if (!illegalComponents.isEmpty()) { @@ -114,15 +114,25 @@ public static void validateUrn( } /** Recursively process URN parts with URL decoding */ - private static Stream processUrnPartRecursively(String urnPart) { + private static Stream processUrnPartRecursively(String urnPart, int totalParts) { String decodedPart = URLDecoder.decode(URLEncodingFixer.fixURLEncoding(urnPart), StandardCharsets.UTF_8); if (decodedPart.startsWith("urn:li:")) { // Recursively process nested URN after decoding + int nestedParts = UrnUtils.getUrn(decodedPart).getEntityKey().getParts().size(); return UrnUtils.getUrn(decodedPart).getEntityKey().getParts().stream() - .flatMap(ValidationApiUtils::processUrnPartRecursively); + .flatMap(part -> processUrnPartRecursively(part, nestedParts)); } - return Stream.of(decodedPart); + if (totalParts > 1) { + if (ILLEGAL_URN_TUPLE_CHARACTERS.stream().anyMatch(c -> urnPart.contains(c))) { + return Stream.of(urnPart); + } + } + if (ILLEGAL_URN_COMPONENT_CHARACTERS.stream().anyMatch(c -> urnPart.contains(c))) { + return Stream.of(urnPart); + } + + return Stream.empty(); } /** diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java index e683e594d8766c..a2c9a15d92f90a 100644 --- a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java +++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java @@ -18,10 +18,36 @@ public void testValidateDatasetUrn() { // If no exception is thrown, test passes } - @Test(expectedExceptions = IllegalArgumentException.class) + @Test public void testSimpleUrnColon() { - Urn invalidUrn = UrnUtils.getUrn("urn:li:corpuser:foo:bar"); - ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + ValidationApiUtils.validateUrn( + entityRegistry, UrnUtils.getUrn("urn:li:corpuser:foo:bar"), true); + ValidationApiUtils.validateUrn( + entityRegistry, UrnUtils.getUrn("urn:li:dataPlatform:abc:def"), true); + ValidationApiUtils.validateUrn( + entityRegistry, UrnUtils.getUrn("urn:li:corpuser:foo:bar@example.com"), true); + // If no exception is thrown, test passes + } + + @Test + public void testSimpleUrnComma() { + ValidationApiUtils.validateUrn(entityRegistry, UrnUtils.getUrn("urn:li:corpuser:,"), true); + // If no exception is thrown, test passes + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testTupleUrnComma() { + ValidationApiUtils.validateUrn( + entityRegistry, UrnUtils.getUrn("urn:li:dashboard:(looker,dashboards,thelook)"), true); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testFabricTypeCasing() { + // prod != PROD + ValidationApiUtils.validateUrn( + entityRegistry, + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:abc:def,table_name,prod)"), + true); } @Test @@ -34,7 +60,7 @@ public void testComplexUrnColon() throws URISyntaxException { } @Test(expectedExceptions = IllegalArgumentException.class) - public void testUrnFabricType() { + public void testFabricTypeParen() { Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,/path/to/data,())"); ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); } @@ -83,20 +109,20 @@ public void testValidComplexUrn() { UrnUtils.getUrn( "urn:li:dataset:(urn:li:dataPlatform:bigquery,myproject.dataset.table,PROD)"); - ValidationApiUtils.validateUrn(entityRegistry, validUrn); + ValidationApiUtils.validateUrn(entityRegistry, validUrn, true); // If no exception is thrown, test passes } @Test(expectedExceptions = NullPointerException.class) public void testUrnNull() { - ValidationApiUtils.validateUrn(entityRegistry, null); + ValidationApiUtils.validateUrn(entityRegistry, null, true); } @Test public void testValidPartialUrlEncode() { Urn validUrn = UrnUtils.getUrn("urn:li:assertion:123=-%28__% weekly__%29"); - ValidationApiUtils.validateUrn(entityRegistry, validUrn); + ValidationApiUtils.validateUrn(entityRegistry, validUrn, true); // If no exception is thrown, test passes } @@ -106,7 +132,23 @@ public void testValidPartialUrlEncode2() { UrnUtils.getUrn( "urn:li:dataset:(urn:li:dataPlatform:s3,urn:li:dataset:%28urn:li:dataPlatform:s3%2Ctest-datalake-concepts%prog_maintenance%2CPROD%29,PROD)"); - ValidationApiUtils.validateUrn(entityRegistry, validUrn); + ValidationApiUtils.validateUrn(entityRegistry, validUrn, true); + // If no exception is thrown, test passes + } + + @Test + public void testValidColon() { + Urn validUrn = + UrnUtils.getUrn("urn:li:dashboard:(looker,dashboards.thelook::cohort_data_tool)"); + + ValidationApiUtils.validateUrn(entityRegistry, validUrn, true); + // If no exception is thrown, test passes + } + + @Test + public void testNoTupleComma() { + Urn invalidUrn = UrnUtils.getUrn("urn:li:corpuser:,"); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); // If no exception is thrown, test passes } } From 823207668b9005f949ac7c89aa36164a1040ed94 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 27 Nov 2024 22:22:55 -0600 Subject: [PATCH 092/174] feat(search): Update search_config.yaml (#11971) --- .../configuration/src/main/resources/search_config.yaml | 4 ++-- smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-service/configuration/src/main/resources/search_config.yaml b/metadata-service/configuration/src/main/resources/search_config.yaml index e93f8af8b1d6c4..47494c8cb1ca43 100644 --- a/metadata-service/configuration/src/main/resources/search_config.yaml +++ b/metadata-service/configuration/src/main/resources/search_config.yaml @@ -65,9 +65,9 @@ queryConfigurations: boost_mode: replace # Criteria for exact-match only - # Contains quotes, is a single term with `_`, `.`, or `-` (normally consider for tokenization) then use exact match query + # Contains quotes then use exact match query - queryRegex: >- - ^["'].+["']$|^[a-zA-Z0-9]\S+[_.-]\S+[a-zA-Z0-9]$ + ^["'].+["']$ simpleQuery: false prefixMatchQuery: true exactMatchQuery: true diff --git a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js index fb772bd7af1e74..57617d7721e594 100644 --- a/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js +++ b/smoke-test/tests/cypress/cypress/e2e/siblings/siblings.js @@ -98,7 +98,7 @@ describe("siblings", () => { it("will combine results in search", () => { cy.login(); - cy.visit("/search?page=1&query=raw_orders"); + cy.visit("/search?page=1&query=%22raw_orders%22"); cy.contains("Showing 1 - 2 of "); From 2206e58d4c91da72dc0c300baa78f70cf02a6329 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Thu, 28 Nov 2024 18:53:23 +0530 Subject: [PATCH 093/174] docs(release): update recommended CLI (#11986) --- docs/managed-datahub/release-notes/v_0_3_7.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index 694283d945facc..94cbdd79dbf5ef 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -7,7 +7,7 @@ Release Availability Date Recommended CLI/SDK --- -- `v0.14.1.11` with release notes at https://github.com/datahub/datahub/releases/tag/v0.14.1.11 +- `v0.14.1.12` with release notes at https://github.com/datahub/datahub/releases/tag/v0.14.1.12 If you are using an older CLI/SDK version, then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, GitHub Actions, Airflow, in Python SDK somewhere, Java SDK, etc. This is a strong recommendation to upgrade, as we keep on pushing fixes in the CLI, and it helps us support you better. @@ -116,7 +116,7 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies - Improved UX for setting up and managing SSO - Ingestion changes - - In addition to the improvements listed here: https://github.com/acryldata/datahub/releases/tag/v0.14.1.11 + - In addition to the improvements listed here: https://github.com/acryldata/datahub/releases/tag/v0.14.1.12 - PowerBI: Support for PowerBI Apps and cross-workspace lineage - Fivetran: Major improvements to configurability and improved reliability with large Fivetran setups - Snowflake & BigQuery: Improved handling of temporary tables and swap statements when generating lineage From ecba2244f04b41384c35344b0532ccca3ad43aa7 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 28 Nov 2024 19:23:30 +0530 Subject: [PATCH 094/174] fix(ingest/kafka):add poll for admin client for oauth_cb (#11985) Co-authored-by: Tamas Nemeth --- .../datahub/ingestion/source/kafka/kafka.py | 29 +++++++++++----- .../tests/integration/kafka/test_kafka.py | 33 +++++++++++++++---- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py index e57dc853a83c65..709ba431f0f87b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py @@ -148,7 +148,7 @@ def get_kafka_consumer( ) -> confluent_kafka.Consumer: consumer = confluent_kafka.Consumer( { - "group.id": "test", + "group.id": "datahub-kafka-ingestion", "bootstrap.servers": connection.bootstrap, **connection.consumer_config, } @@ -164,6 +164,25 @@ def get_kafka_consumer( return consumer +def get_kafka_admin_client( + connection: KafkaConsumerConnectionConfig, +) -> AdminClient: + client = AdminClient( + { + "group.id": "datahub-kafka-ingestion", + "bootstrap.servers": connection.bootstrap, + **connection.consumer_config, + } + ) + if CallableConsumerConfig.is_callable_config(connection.consumer_config): + # As per documentation, we need to explicitly call the poll method to make sure OAuth callback gets executed + # https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#kafka-client-configuration + logger.debug("Initiating polling for kafka admin client") + client.poll(timeout=30) + logger.debug("Initiated polling for kafka admin client") + return client + + @dataclass class KafkaSourceReport(StaleEntityRemovalSourceReport): topics_scanned: int = 0 @@ -278,13 +297,7 @@ def __init__(self, config: KafkaSourceConfig, ctx: PipelineContext): def init_kafka_admin_client(self) -> None: try: # TODO: Do we require separate config than existing consumer_config ? - self.admin_client = AdminClient( - { - "group.id": "test", - "bootstrap.servers": self.source_config.connection.bootstrap, - **self.source_config.connection.consumer_config, - } - ) + self.admin_client = get_kafka_admin_client(self.source_config.connection) except Exception as e: logger.debug(e, exc_info=e) self.report.report_warning( diff --git a/metadata-ingestion/tests/integration/kafka/test_kafka.py b/metadata-ingestion/tests/integration/kafka/test_kafka.py index 597889c8440b7a..7462f177684b7e 100644 --- a/metadata-ingestion/tests/integration/kafka/test_kafka.py +++ b/metadata-ingestion/tests/integration/kafka/test_kafka.py @@ -128,11 +128,32 @@ def test_kafka_oauth_callback( pipeline.run() - is_found: bool = False + # Initialize flags to track oauth events + checks = { + "consumer_polling": False, + "consumer_oauth_callback": False, + "admin_polling": False, + "admin_oauth_callback": False, + } + + # Read log file and check for oauth events with open(log_file, "r") as file: - for line_number, line in enumerate(file, 1): + for line in file: + # Check for polling events + if "Initiating polling for kafka admin client" in line: + checks["admin_polling"] = True + elif "Initiating polling for kafka consumer" in line: + checks["consumer_polling"] = True + + # Check for oauth callbacks if oauth.MESSAGE in line: - is_found = True - break - - assert is_found + if checks["consumer_polling"] and not checks["admin_polling"]: + checks["consumer_oauth_callback"] = True + elif checks["consumer_polling"] and checks["admin_polling"]: + checks["admin_oauth_callback"] = True + + # Verify all oauth events occurred + assert checks["consumer_polling"], "Consumer polling was not initiated" + assert checks["consumer_oauth_callback"], "Consumer oauth callback not found" + assert checks["admin_polling"], "Admin polling was not initiated" + assert checks["admin_oauth_callback"], "Admin oauth callback not found" From 816fd3dba7fe2ca7276cb3b0b8042b74c0d8e04c Mon Sep 17 00:00:00 2001 From: skrydal Date: Thu, 28 Nov 2024 16:34:20 +0100 Subject: [PATCH 095/174] fix(ingestion/iceberg): Improvements to iceberg source (#11987) --- .../src/app/ingest/source/builder/sources.json | 8 ++++++++ .../docs/sources/iceberg/iceberg.md | 6 ++++-- metadata-ingestion/setup.py | 3 ++- .../datahub/ingestion/source/iceberg/iceberg.py | 17 ++++++++++++----- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 5e81a7855e9c4c..70d9baabdb4bc6 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -317,5 +317,13 @@ "displayName": "CassandraDB", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/cassandra", "recipe": "source:\n type: cassandra\n config:\n # Credentials for on prem cassandra\n contact_point: localhost\n port: 9042\n username: admin\n password: password\n\n # Or\n # Credentials Astra Cloud\n #cloud_config:\n # secure_connect_bundle: Path to Secure Connect Bundle (.zip)\n # token: Application Token\n\n # Optional Allow / Deny extraction of particular keyspaces.\n keyspace_pattern:\n allow: [.*]\n\n # Optional Allow / Deny extraction of particular tables.\n table_pattern:\n allow: [.*]" + }, + { + "urn": "urn:li:dataPlatform:iceberg", + "name": "iceberg", + "displayName": "Iceberg", + "description": "Ingest databases and tables from any Iceberg catalog implementation", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/iceberg", + "recipe": "source:\n type: \"iceberg\"\n config:\n env: dev\n # each thread will open internet connections to fetch manifest files independently, \n # this value needs to be adjusted with ulimit\n processing_threads: 1 \n # a single catalog definition with a form of a dictionary\n catalog: \n demo: # name of the catalog\n type: \"rest\" # other types are available\n uri: \"uri\"\n s3.access-key-id: \"access-key\"\n s3.secret-access-key: \"secret-access-key\"\n s3.region: \"aws-region\"\n profiling:\n enabled: false\n" } ] diff --git a/metadata-ingestion/docs/sources/iceberg/iceberg.md b/metadata-ingestion/docs/sources/iceberg/iceberg.md index 7e40315a2e3193..92aac5ffa6ce51 100644 --- a/metadata-ingestion/docs/sources/iceberg/iceberg.md +++ b/metadata-ingestion/docs/sources/iceberg/iceberg.md @@ -18,6 +18,8 @@ This ingestion source maps the following Source System Concepts to DataHub Conce ## Troubleshooting -### [Common Issue] +### Exceptions while increasing `processing_threads` -[Provide description of common issues with this integration and steps to resolve] +Each processing thread will open several files/sockets to download manifest files from blob storage. If you experience +exceptions appearing when increasing `processing_threads` configuration parameter, try to increase limit of open +files (i.e. using `ulimit` in Linux). diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 74c2e611cf68f1..292038380e6a22 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -249,7 +249,8 @@ iceberg_common = { # Iceberg Python SDK - "pyiceberg>=0.4,<0.7", + # Kept at 0.4.0 due to higher versions requiring pydantic>2, as soon as we are fine with it, bump this dependency + "pyiceberg>=0.4.0", } mssql_common = { diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py index 258a4b9ad6daf6..5931873f54236d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py @@ -9,6 +9,7 @@ NoSuchIcebergTableError, NoSuchNamespaceError, NoSuchPropertyException, + NoSuchTableError, ) from pyiceberg.schema import Schema, SchemaVisitorPerPrimitiveType, visit from pyiceberg.table import Table @@ -104,7 +105,7 @@ @capability(SourceCapability.DESCRIPTIONS, "Enabled by default.") @capability( SourceCapability.OWNERSHIP, - "Optionally enabled via configuration by specifying which Iceberg table property holds user or group ownership.", + "Automatically ingests ownership information from table properties based on `user_ownership_property` and `group_ownership_property`", ) @capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion") class IcebergSource(StatefulIngestionSourceBase): @@ -192,9 +193,7 @@ def _process_dataset(dataset_path: Identifier) -> Iterable[MetadataWorkUnit]: table = thread_local.local_catalog.load_table(dataset_path) time_taken = timer.elapsed_seconds() self.report.report_table_load_time(time_taken) - LOGGER.debug( - f"Loaded table: {table.identifier}, time taken: {time_taken}" - ) + LOGGER.debug(f"Loaded table: {table.name()}, time taken: {time_taken}") yield from self._create_iceberg_workunit(dataset_name, table) except NoSuchPropertyException as e: self.report.report_warning( @@ -206,12 +205,20 @@ def _process_dataset(dataset_path: Identifier) -> Iterable[MetadataWorkUnit]: ) except NoSuchIcebergTableError as e: self.report.report_warning( - "no-iceberg-table", + "not-an-iceberg-table", f"Failed to create workunit for {dataset_name}. {e}", ) LOGGER.warning( f"NoSuchIcebergTableError while processing table {dataset_path}, skipping it.", ) + except NoSuchTableError as e: + self.report.report_warning( + "no-such-table", + f"Failed to create workunit for {dataset_name}. {e}", + ) + LOGGER.warning( + f"NoSuchTableError while processing table {dataset_path}, skipping it.", + ) except Exception as e: self.report.report_failure("general", f"Failed to create workunit: {e}") LOGGER.exception( From 189f8cefa71b1137801368edf26f633c80599016 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 28 Nov 2024 14:53:46 -0500 Subject: [PATCH 096/174] feat(ingest): standardize sql type mappings (#11982) --- .../ingestion/source/dbt/dbt_common.py | 68 ++-------------- .../source/qlik_sense/data_classes.py | 1 + .../ingestion/source/redshift/redshift.py | 1 + .../source/snowflake/snowflake_schema_gen.py | 1 + .../datahub/ingestion/source/sql/sql_types.py | 79 +++++++++++++++++-- .../ingestion/source/unity/proxy_types.py | 1 + .../tests/integration/dbt/test_dbt.py | 69 ---------------- .../tests/unit/test_sql_types.py | 78 ++++++++++++++++++ 8 files changed, 161 insertions(+), 137 deletions(-) create mode 100644 metadata-ingestion/tests/unit/test_sql_types.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 4598ae388b827d..499e7e1231d050 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -53,19 +53,7 @@ make_assertion_from_test, make_assertion_result_from_test, ) -from datahub.ingestion.source.sql.sql_types import ( - ATHENA_SQL_TYPES_MAP, - BIGQUERY_TYPES_MAP, - POSTGRES_TYPES_MAP, - SNOWFLAKE_TYPES_MAP, - SPARK_SQL_TYPES_MAP, - TRINO_SQL_TYPES_MAP, - VERTICA_SQL_TYPES_MAP, - resolve_athena_modified_type, - resolve_postgres_modified_type, - resolve_trino_modified_type, - resolve_vertica_modified_type, -) +from datahub.ingestion.source.sql.sql_types import resolve_sql_type from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, StaleEntityRemovalSourceReport, @@ -89,17 +77,11 @@ from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.com.linkedin.pegasus2avro.schema import ( - BooleanTypeClass, - DateTypeClass, MySqlDDL, NullTypeClass, - NumberTypeClass, - RecordType, SchemaField, SchemaFieldDataType, SchemaMetadata, - StringTypeClass, - TimeTypeClass, ) from datahub.metadata.schema_classes import ( DataPlatformInstanceClass, @@ -804,28 +786,6 @@ def make_mapping_upstream_lineage( ) -# See https://github.com/fishtown-analytics/dbt/blob/master/core/dbt/adapters/sql/impl.py -_field_type_mapping = { - "boolean": BooleanTypeClass, - "date": DateTypeClass, - "time": TimeTypeClass, - "numeric": NumberTypeClass, - "text": StringTypeClass, - "timestamp with time zone": DateTypeClass, - "timestamp without time zone": DateTypeClass, - "integer": NumberTypeClass, - "float8": NumberTypeClass, - "struct": RecordType, - **POSTGRES_TYPES_MAP, - **SNOWFLAKE_TYPES_MAP, - **BIGQUERY_TYPES_MAP, - **SPARK_SQL_TYPES_MAP, - **TRINO_SQL_TYPES_MAP, - **ATHENA_SQL_TYPES_MAP, - **VERTICA_SQL_TYPES_MAP, -} - - def get_column_type( report: DBTSourceReport, dataset_name: str, @@ -835,24 +795,10 @@ def get_column_type( """ Maps known DBT types to datahub types """ - TypeClass: Any = _field_type_mapping.get(column_type) if column_type else None - - if TypeClass is None and column_type: - # resolve a modified type - if dbt_adapter == "trino": - TypeClass = resolve_trino_modified_type(column_type) - elif dbt_adapter == "athena": - TypeClass = resolve_athena_modified_type(column_type) - elif dbt_adapter == "postgres" or dbt_adapter == "redshift": - # Redshift uses a variant of Postgres, so we can use the same logic. - TypeClass = resolve_postgres_modified_type(column_type) - elif dbt_adapter == "vertica": - TypeClass = resolve_vertica_modified_type(column_type) - elif dbt_adapter == "snowflake": - # Snowflake types are uppercase, so we check that. - TypeClass = _field_type_mapping.get(column_type.upper()) - - # if still not found, report the warning + + TypeClass = resolve_sql_type(column_type, dbt_adapter) + + # if still not found, report a warning if TypeClass is None: if column_type: report.info( @@ -861,9 +807,9 @@ def get_column_type( context=f"{dataset_name} - {column_type}", log=False, ) - TypeClass = NullTypeClass + TypeClass = NullTypeClass() - return SchemaFieldDataType(type=TypeClass()) + return SchemaFieldDataType(type=TypeClass) @platform_name("dbt") diff --git a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py index 672fcbceb0603b..a43f5f32493f2d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/qlik_sense/data_classes.py @@ -15,6 +15,7 @@ TimeType, ) +# TODO: Replace with standardized types in sql_types.py FIELD_TYPE_MAPPING: Dict[ str, Type[ diff --git a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py index 4bc4c1451c262f..06cbb7fbae27cc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py @@ -222,6 +222,7 @@ class RedshiftSource(StatefulIngestionSourceBase, TestableSource): ``` """ + # TODO: Replace with standardized types in sql_types.py REDSHIFT_FIELD_TYPE_MAPPINGS: Dict[ str, Type[ diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py index d4442749a06224..2bd8e8017f5492 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py @@ -103,6 +103,7 @@ logger = logging.getLogger(__name__) # https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html +# TODO: Move to the standardized types in sql_types.py SNOWFLAKE_FIELD_TYPE_MAPPINGS = { "DATE": DateType, "BIGINT": NumberType, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py index 8ea4209784063f..89ca160ba1f487 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py @@ -1,5 +1,5 @@ import re -from typing import Any, Dict, ValuesView +from typing import Any, Dict, Optional, Type, Union, ValuesView from datahub.metadata.com.linkedin.pegasus2avro.schema import ( ArrayType, @@ -16,14 +16,28 @@ UnionType, ) -# these can be obtained by running `select format_type(oid, null),* from pg_type;` -# we've omitted the types without a meaningful DataHub type (e.g. postgres-specific types, index vectors, etc.) -# (run `\copy (select format_type(oid, null),* from pg_type) to 'pg_type.csv' csv header;` to get a CSV) +DATAHUB_FIELD_TYPE = Union[ + ArrayType, + BooleanType, + BytesType, + DateType, + EnumType, + MapType, + NullType, + NumberType, + RecordType, + StringType, + TimeType, + UnionType, +] -# we map from format_type since this is what dbt uses -# see https://github.com/fishtown-analytics/dbt/blob/master/plugins/postgres/dbt/include/postgres/macros/catalog.sql#L22 -# see https://www.npgsql.org/dev/types.html for helpful type annotations +# These can be obtained by running `select format_type(oid, null),* from pg_type;` +# We've omitted the types without a meaningful DataHub type (e.g. postgres-specific types, index vectors, etc.) +# (run `\copy (select format_type(oid, null),* from pg_type) to 'pg_type.csv' csv header;` to get a CSV) +# We map from format_type since this is what dbt uses. +# See https://github.com/fishtown-analytics/dbt/blob/master/plugins/postgres/dbt/include/postgres/macros/catalog.sql#L22 +# See https://www.npgsql.org/dev/types.html for helpful type annotations POSTGRES_TYPES_MAP: Dict[str, Any] = { "boolean": BooleanType, "bytea": BytesType, @@ -430,3 +444,54 @@ def resolve_vertica_modified_type(type_string: str) -> Any: "geography": None, "uuid": StringType, } + + +_merged_mapping = { + "boolean": BooleanType, + "date": DateType, + "time": TimeType, + "numeric": NumberType, + "text": StringType, + "timestamp with time zone": DateType, + "timestamp without time zone": DateType, + "integer": NumberType, + "float8": NumberType, + "struct": RecordType, + **POSTGRES_TYPES_MAP, + **SNOWFLAKE_TYPES_MAP, + **BIGQUERY_TYPES_MAP, + **SPARK_SQL_TYPES_MAP, + **TRINO_SQL_TYPES_MAP, + **ATHENA_SQL_TYPES_MAP, + **VERTICA_SQL_TYPES_MAP, +} + + +def resolve_sql_type( + column_type: Optional[str], + platform: Optional[str] = None, +) -> Optional[DATAHUB_FIELD_TYPE]: + # In theory, we should use the platform-specific mapping where available. + # However, the types don't ever conflict, so the merged mapping is fine. + TypeClass: Optional[Type[DATAHUB_FIELD_TYPE]] = ( + _merged_mapping.get(column_type) if column_type else None + ) + + if TypeClass is None and column_type: + # resolve a modified type + if platform == "trino": + TypeClass = resolve_trino_modified_type(column_type) + elif platform == "athena": + TypeClass = resolve_athena_modified_type(column_type) + elif platform == "postgres" or platform == "redshift": + # Redshift uses a variant of Postgres, so we can use the same logic. + TypeClass = resolve_postgres_modified_type(column_type) + elif platform == "vertica": + TypeClass = resolve_vertica_modified_type(column_type) + elif platform == "snowflake": + # Snowflake types are uppercase, so we check that. + TypeClass = _merged_mapping.get(column_type.upper()) + + if TypeClass: + return TypeClass() + return None diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py index f84f6c1b0c08d6..9c5752c518df14 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy_types.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) +# TODO: (maybe) Replace with standardized types in sql_types.py DATA_TYPE_REGISTRY: dict = { ColumnTypeName.BOOLEAN: BooleanTypeClass, ColumnTypeName.BYTE: BytesTypeClass, diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 390d8d7698dd4c..c6a3dc4fd590bd 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -11,12 +11,6 @@ from datahub.ingestion.run.pipeline_config import PipelineConfig, SourceConfig from datahub.ingestion.source.dbt.dbt_common import DBTEntitiesEnabled, EmitDirective from datahub.ingestion.source.dbt.dbt_core import DBTCoreConfig, DBTCoreSource -from datahub.ingestion.source.sql.sql_types import ( - ATHENA_SQL_TYPES_MAP, - TRINO_SQL_TYPES_MAP, - resolve_athena_modified_type, - resolve_trino_modified_type, -) from tests.test_helpers import mce_helpers, test_connection_helpers FROZEN_TIME = "2022-02-03 07:00:00" @@ -362,69 +356,6 @@ def test_dbt_tests(test_resources_dir, pytestconfig, tmp_path, mock_time, **kwar ) -@pytest.mark.parametrize( - "data_type, expected_data_type", - [ - ("boolean", "boolean"), - ("tinyint", "tinyint"), - ("smallint", "smallint"), - ("int", "int"), - ("integer", "integer"), - ("bigint", "bigint"), - ("real", "real"), - ("double", "double"), - ("decimal(10,0)", "decimal"), - ("varchar(20)", "varchar"), - ("char", "char"), - ("varbinary", "varbinary"), - ("json", "json"), - ("date", "date"), - ("time", "time"), - ("time(12)", "time"), - ("timestamp", "timestamp"), - ("timestamp(3)", "timestamp"), - ("row(x bigint, y double)", "row"), - ("array(row(x bigint, y double))", "array"), - ("map(varchar, varchar)", "map"), - ], -) -def test_resolve_trino_modified_type(data_type, expected_data_type): - assert ( - resolve_trino_modified_type(data_type) - == TRINO_SQL_TYPES_MAP[expected_data_type] - ) - - -@pytest.mark.parametrize( - "data_type, expected_data_type", - [ - ("boolean", "boolean"), - ("tinyint", "tinyint"), - ("smallint", "smallint"), - ("int", "int"), - ("integer", "integer"), - ("bigint", "bigint"), - ("float", "float"), - ("double", "double"), - ("decimal(10,0)", "decimal"), - ("varchar(20)", "varchar"), - ("char", "char"), - ("binary", "binary"), - ("date", "date"), - ("timestamp", "timestamp"), - ("timestamp(3)", "timestamp"), - ("struct", "struct"), - ("array>", "array"), - ("map", "map"), - ], -) -def test_resolve_athena_modified_type(data_type, expected_data_type): - assert ( - resolve_athena_modified_type(data_type) - == ATHENA_SQL_TYPES_MAP[expected_data_type] - ) - - @pytest.mark.integration @freeze_time(FROZEN_TIME) def test_dbt_tests_only_assertions( diff --git a/metadata-ingestion/tests/unit/test_sql_types.py b/metadata-ingestion/tests/unit/test_sql_types.py new file mode 100644 index 00000000000000..ebe5ade115cdd4 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_sql_types.py @@ -0,0 +1,78 @@ +import pytest + +from datahub.ingestion.source.sql.sql_types import ( + ATHENA_SQL_TYPES_MAP, + TRINO_SQL_TYPES_MAP, + resolve_athena_modified_type, + resolve_sql_type, + resolve_trino_modified_type, +) +from datahub.metadata.schema_classes import BooleanTypeClass, StringTypeClass + + +@pytest.mark.parametrize( + "data_type, expected_data_type", + [ + ("boolean", "boolean"), + ("tinyint", "tinyint"), + ("smallint", "smallint"), + ("int", "int"), + ("integer", "integer"), + ("bigint", "bigint"), + ("real", "real"), + ("double", "double"), + ("decimal(10,0)", "decimal"), + ("varchar(20)", "varchar"), + ("char", "char"), + ("varbinary", "varbinary"), + ("json", "json"), + ("date", "date"), + ("time", "time"), + ("time(12)", "time"), + ("timestamp", "timestamp"), + ("timestamp(3)", "timestamp"), + ("row(x bigint, y double)", "row"), + ("array(row(x bigint, y double))", "array"), + ("map(varchar, varchar)", "map"), + ], +) +def test_resolve_trino_modified_type(data_type, expected_data_type): + assert ( + resolve_trino_modified_type(data_type) + == TRINO_SQL_TYPES_MAP[expected_data_type] + ) + + +@pytest.mark.parametrize( + "data_type, expected_data_type", + [ + ("boolean", "boolean"), + ("tinyint", "tinyint"), + ("smallint", "smallint"), + ("int", "int"), + ("integer", "integer"), + ("bigint", "bigint"), + ("float", "float"), + ("double", "double"), + ("decimal(10,0)", "decimal"), + ("varchar(20)", "varchar"), + ("char", "char"), + ("binary", "binary"), + ("date", "date"), + ("timestamp", "timestamp"), + ("timestamp(3)", "timestamp"), + ("struct", "struct"), + ("array>", "array"), + ("map", "map"), + ], +) +def test_resolve_athena_modified_type(data_type, expected_data_type): + assert ( + resolve_athena_modified_type(data_type) + == ATHENA_SQL_TYPES_MAP[expected_data_type] + ) + + +def test_resolve_sql_type() -> None: + assert resolve_sql_type("boolean") == BooleanTypeClass() + assert resolve_sql_type("varchar") == StringTypeClass() From 0476bf1e32d8892aef4faf493a2703a34bad9d48 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 28 Nov 2024 21:41:53 -0500 Subject: [PATCH 097/174] feat(ingest): bump typing_extensions dep (#11965) --- metadata-ingestion/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 292038380e6a22..d7e056b31370df 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -14,8 +14,8 @@ ) base_requirements = { - # Typing extension should be >=3.10.0.2 ideally but we can't restrict due to a Airflow 2.1 dependency conflict. - "typing_extensions>=3.7.4.3", + # Our min version of typing_extensions is somewhat constrained by Airflow. + "typing_extensions>=3.10.0.2", # Actual dependencies. "typing-inspect", # pydantic 1.8.2 is incompatible with mypy 0.910. From a92c6b2bb0768b33ba479b012d51ef20ae2194f2 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 28 Nov 2024 21:42:40 -0500 Subject: [PATCH 098/174] feat(ingest): add tests for colon characters in urns (#11976) --- .../src/datahub/utilities/urn_encoder.py | 3 +- .../tests/unit/urns/test_urn.py | 49 ++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/utilities/urn_encoder.py b/metadata-ingestion/src/datahub/utilities/urn_encoder.py index 88c0a128b8e468..4f19eeff3e70f0 100644 --- a/metadata-ingestion/src/datahub/utilities/urn_encoder.py +++ b/metadata-ingestion/src/datahub/utilities/urn_encoder.py @@ -4,7 +4,8 @@ # NOTE: Frontend relies on encoding these three characters. Specifically, we decode and encode schema fields for column level lineage. # If this changes, make appropriate changes to datahub-web-react/src/app/lineage/utils/columnLineageUtils.ts # We also rely on encoding these exact three characters when generating schemaField urns in our graphQL layer. Update SchemaFieldUtils if this changes. -RESERVED_CHARS = {",", "(", ")"} +# Also see https://datahubproject.io/docs/what/urn/#restrictions +RESERVED_CHARS = {",", "(", ")", "␟"} RESERVED_CHARS_EXTENDED = RESERVED_CHARS.union({"%"}) diff --git a/metadata-ingestion/tests/unit/urns/test_urn.py b/metadata-ingestion/tests/unit/urns/test_urn.py index 1bf48082fec8c9..73badb3d1b4234 100644 --- a/metadata-ingestion/tests/unit/urns/test_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_urn.py @@ -1,6 +1,12 @@ import pytest -from datahub.metadata.urns import DatasetUrn, Urn +from datahub.metadata.urns import ( + CorpUserUrn, + DashboardUrn, + DataPlatformUrn, + DatasetUrn, + Urn, +) from datahub.utilities.urns.error import InvalidUrnError pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -36,20 +42,51 @@ def test_url_encode_urn() -> None: def test_invalid_urn() -> None: with pytest.raises(InvalidUrnError): - Urn.create_from_string("urn:li:abc") + Urn.from_string("urn:li:abc") with pytest.raises(InvalidUrnError): - Urn.create_from_string("urn:li:abc:") + Urn.from_string("urn:li:abc:") with pytest.raises(InvalidUrnError): - Urn.create_from_string("urn:li:abc:()") + Urn.from_string("urn:li:abc:()") with pytest.raises(InvalidUrnError): - Urn.create_from_string("urn:li:abc:(abc,)") + Urn.from_string("urn:li:abc:(abc,)") + + with pytest.raises(InvalidUrnError): + Urn.from_string("urn:li:corpuser:abc)") + + +def test_urn_colon() -> None: + # Colon characters are valid in urns, and should not mess up parsing. + + urn = Urn.from_string( + "urn:li:dashboard:(looker,dashboards.thelook::customer_lookup)" + ) + assert isinstance(urn, DashboardUrn) + + assert DataPlatformUrn.from_string("urn:li:dataPlatform:abc:def") + assert DatasetUrn.from_string( + "urn:li:dataset:(urn:li:dataPlatform:abc:def,table_name,PROD)" + ) + assert Urn.from_string("urn:li:corpuser:foo:bar@example.com") + + # I'm not sure why you'd ever want this, but technically it's a valid urn. + urn = Urn.from_string("urn:li:corpuser::") + assert isinstance(urn, CorpUserUrn) + assert urn.username == ":" + assert urn == CorpUserUrn(":") + + +def test_urn_coercion() -> None: + urn = CorpUserUrn("foo␟bar") + assert urn.urn() == "urn:li:corpuser:foo%E2%90%9Fbar" + + assert urn == Urn.from_string(urn.urn()) def test_urn_type_dispatch() -> None: - urn = Urn.from_string("urn:li:dataset:(urn:li:dataPlatform:abc,def,prod)") + urn = Urn.from_string("urn:li:dataset:(urn:li:dataPlatform:abc,def,PROD)") assert isinstance(urn, DatasetUrn) with pytest.raises(InvalidUrnError, match="Passed an urn of type corpuser"): From a46de1ecf9b67e3520c991a51c3f0d9a4f9051a1 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 28 Nov 2024 21:42:55 -0500 Subject: [PATCH 099/174] feat(ingest/athena): handle partition fetching errors (#11966) --- .../datahub/ingestion/source/sql/athena.py | 68 +++++++++++++------ .../tests/unit/test_athena_source.py | 45 +++++++++++- 2 files changed, 88 insertions(+), 25 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index 71cfd0268ee6b5..6f7decc79b1df2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -26,6 +26,7 @@ platform_name, support_status, ) +from datahub.ingestion.api.source import StructuredLogLevel from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws.s3_util import make_s3_urn from datahub.ingestion.source.common.subtypes import DatasetContainerSubTypes @@ -35,6 +36,7 @@ register_custom_type, ) from datahub.ingestion.source.sql.sql_config import SQLCommonConfig, make_sqlalchemy_uri +from datahub.ingestion.source.sql.sql_report import SQLSourceReport from datahub.ingestion.source.sql.sql_utils import ( add_table_to_schema_container, gen_database_container, @@ -48,6 +50,15 @@ get_schema_fields_for_sqlalchemy_column, ) +try: + from typing_extensions import override +except ImportError: + _F = typing.TypeVar("_F", bound=typing.Callable[..., typing.Any]) + + def override(f: _F, /) -> _F: # noqa: F811 + return f + + logger = logging.getLogger(__name__) assert STRUCT, "required type modules are not available" @@ -322,12 +333,15 @@ class AthenaSource(SQLAlchemySource): - Profiling when enabled. """ - table_partition_cache: Dict[str, Dict[str, Partitionitem]] = {} + config: AthenaConfig + report: SQLSourceReport def __init__(self, config, ctx): super().__init__(config, ctx, "athena") self.cursor: Optional[BaseCursor] = None + self.table_partition_cache: Dict[str, Dict[str, Partitionitem]] = {} + @classmethod def create(cls, config_dict, ctx): config = AthenaConfig.parse_obj(config_dict) @@ -452,6 +466,7 @@ def add_table_to_schema_container( ) # It seems like database/schema filter in the connection string does not work and this to work around that + @override def get_schema_names(self, inspector: Inspector) -> List[str]: athena_config = typing.cast(AthenaConfig, self.config) schemas = inspector.get_schema_names() @@ -459,34 +474,42 @@ def get_schema_names(self, inspector: Inspector) -> List[str]: return [schema for schema in schemas if schema == athena_config.database] return schemas - # Overwrite to get partitions + @classmethod + def _casted_partition_key(cls, key: str) -> str: + # We need to cast the partition keys to a VARCHAR, since otherwise + # Athena may throw an error during concatenation / comparison. + return f"CAST({key} as VARCHAR)" + + @override def get_partitions( self, inspector: Inspector, schema: str, table: str - ) -> List[str]: - partitions = [] - - athena_config = typing.cast(AthenaConfig, self.config) - - if not athena_config.extract_partitions: - return [] + ) -> Optional[List[str]]: + if not self.config.extract_partitions: + return None if not self.cursor: - return [] + return None metadata: AthenaTableMetadata = self.cursor.get_table_metadata( table_name=table, schema_name=schema ) - if metadata.partition_keys: - for key in metadata.partition_keys: - if key.name: - partitions.append(key.name) - - if not partitions: - return [] + partitions = [] + for key in metadata.partition_keys: + if key.name: + partitions.append(key.name) + if not partitions: + return [] - # We create an artiificaial concatenated partition key to be able to query max partition easier - part_concat = "|| '-' ||".join(partitions) + with self.report.report_exc( + message="Failed to extract partition details", + context=f"{schema}.{table}", + level=StructuredLogLevel.WARN, + ): + # We create an artifical concatenated partition key to be able to query max partition easier + part_concat = " || '-' || ".join( + self._casted_partition_key(key) for key in partitions + ) max_partition_query = f'select {",".join(partitions)} from "{schema}"."{table}$partitions" where {part_concat} = (select max({part_concat}) from "{schema}"."{table}$partitions")' ret = self.cursor.execute(max_partition_query) max_partition: Dict[str, str] = {} @@ -500,9 +523,8 @@ def get_partitions( partitions=partitions, max_partition=max_partition, ) - return partitions - return [] + return partitions # Overwrite to modify the creation of schema fields def get_schema_fields_for_column( @@ -551,7 +573,9 @@ def generate_partition_profiler_query( if partition and partition.max_partition: max_partition_filters = [] for key, value in partition.max_partition.items(): - max_partition_filters.append(f"CAST({key} as VARCHAR) = '{value}'") + max_partition_filters.append( + f"{self._casted_partition_key(key)} = '{value}'" + ) max_partition = str(partition.max_partition) return ( max_partition, diff --git a/metadata-ingestion/tests/unit/test_athena_source.py b/metadata-ingestion/tests/unit/test_athena_source.py index 875cf3800daf88..f8b6220d182735 100644 --- a/metadata-ingestion/tests/unit/test_athena_source.py +++ b/metadata-ingestion/tests/unit/test_athena_source.py @@ -93,7 +93,8 @@ def test_athena_get_table_properties(): "CreateTime": datetime.now(), "LastAccessTime": datetime.now(), "PartitionKeys": [ - {"Name": "testKey", "Type": "string", "Comment": "testComment"} + {"Name": "year", "Type": "string", "Comment": "testComment"}, + {"Name": "month", "Type": "string", "Comment": "testComment"}, ], "Parameters": { "comment": "testComment", @@ -112,8 +113,18 @@ def test_athena_get_table_properties(): response=table_metadata ) + # Mock partition query results + mock_cursor.execute.return_value.description = [ + ["year"], + ["month"], + ] + mock_cursor.execute.return_value.__iter__.return_value = [["2023", "12"]] + ctx = PipelineContext(run_id="test") source = AthenaSource(config=config, ctx=ctx) + source.cursor = mock_cursor + + # Test table properties description, custom_properties, location = source.get_table_properties( inspector=mock_inspector, table=table, schema=schema ) @@ -124,13 +135,35 @@ def test_athena_get_table_properties(): "last_access_time": "2020-04-14 07:00:00", "location": "s3://testLocation", "outputformat": "testOutputFormat", - "partition_keys": '[{"name": "testKey", "type": "string", "comment": "testComment"}]', + "partition_keys": '[{"name": "year", "type": "string", "comment": "testComment"}, {"name": "month", "type": "string", "comment": "testComment"}]', "serde.serialization.lib": "testSerde", "table_type": "testType", } - assert location == make_s3_urn("s3://testLocation", "PROD") + # Test partition functionality + partitions = source.get_partitions( + inspector=mock_inspector, schema=schema, table=table + ) + assert partitions == ["year", "month"] + + # Verify the correct SQL query was generated for partitions + expected_query = """\ +select year,month from "test_schema"."test_table$partitions" \ +where CAST(year as VARCHAR) || '-' || CAST(month as VARCHAR) = \ +(select max(CAST(year as VARCHAR) || '-' || CAST(month as VARCHAR)) \ +from "test_schema"."test_table$partitions")""" + mock_cursor.execute.assert_called_once() + actual_query = mock_cursor.execute.call_args[0][0] + assert actual_query == expected_query + + # Verify partition cache was populated correctly + assert source.table_partition_cache[schema][table].partitions == partitions + assert source.table_partition_cache[schema][table].max_partition == { + "year": "2023", + "month": "12", + } + def test_get_column_type_simple_types(): assert isinstance( @@ -214,3 +247,9 @@ def test_column_type_complex_combination(): assert isinstance( result._STRUCT_fields[2][1].item_type._STRUCT_fields[1][1], types.String ) + + +def test_casted_partition_key(): + from datahub.ingestion.source.sql.athena import AthenaSource + + assert AthenaSource._casted_partition_key("test_col") == "CAST(test_col as VARCHAR)" From c42f77985947418de108e7e5b515aafdbf638ed3 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Fri, 29 Nov 2024 21:28:31 +0530 Subject: [PATCH 100/174] fix: Add option for disabling ownership extraction (#11970) Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../app/ingest/source/builder/sources.json | 2 +- .../docs/sources/dremio/dremio_recipe.yml | 2 ++ .../ingestion/source/dremio/dremio_aspects.py | 34 +++++++++++-------- .../ingestion/source/dremio/dremio_config.py | 5 +++ .../ingestion/source/dremio/dremio_source.py | 2 ++ 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 70d9baabdb4bc6..44b8a37f14655d 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -309,7 +309,7 @@ "displayName": "Dremio", "description": "Import Spaces, Sources, Tables and statistics from Dremio.", "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", - "recipe": "source:\n type: dremio\n config:\n # Coordinates\n hostname: null\n port: null\n #true if https, otherwise false\n tls: true\n\n #For cloud instance\n #is_dremio_cloud: True\n #dremio_cloud_project_id: \n\n #Credentials with personal access token\n authentication_method: PAT\n password: pass\n\n #Or Credentials with basic auth\n #authentication_method: password\n #username: null\n #password: null\n\n stateful_ingestion:\n enabled: true" + "recipe": "source:\n type: dremio\n config:\n # Coordinates\n hostname: null\n port: null\n #true if https, otherwise false\n tls: true\n\n #For cloud instance\n #is_dremio_cloud: True\n #dremio_cloud_project_id: \n\n #Credentials with personal access token\n authentication_method: PAT\n password: pass\n\n #Or Credentials with basic auth\n #authentication_method: password\n #username: null\n #password: null\n\n ingest_owner: true\n\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:cassandra", diff --git a/metadata-ingestion/docs/sources/dremio/dremio_recipe.yml b/metadata-ingestion/docs/sources/dremio/dremio_recipe.yml index 9dcd4f8b337d16..d18d19da2de84b 100644 --- a/metadata-ingestion/docs/sources/dremio/dremio_recipe.yml +++ b/metadata-ingestion/docs/sources/dremio/dremio_recipe.yml @@ -20,6 +20,8 @@ source: include_query_lineage: True + ingest_owner: true + #Optional source_mappings: - platform: s3 diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py index b29fc91a25e74c..d9d85edbf4f7a0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_aspects.py @@ -142,6 +142,7 @@ def __init__( platform: str, ui_url: str, env: str, + ingest_owner: bool, domain: Optional[str] = None, platform_instance: Optional[str] = None, ): @@ -150,6 +151,7 @@ def __init__( self.env = env self.domain = domain self.ui_url = ui_url + self.ingest_owner = ingest_owner def get_container_key( self, name: Optional[str], path: Optional[List[str]] @@ -426,21 +428,23 @@ def _create_external_url(self, dataset: DremioDataset) -> str: return f'{self.ui_url}/{container_type}/{dataset_url_path}"{dataset.resource_name}"' def _create_ownership(self, dataset: DremioDataset) -> Optional[OwnershipClass]: - if not dataset.owner: - return None - owner = ( - make_user_urn(dataset.owner) - if dataset.owner_type == "USER" - else make_group_urn(dataset.owner) - ) - return OwnershipClass( - owners=[ - OwnerClass( - owner=owner, - type=OwnershipTypeClass.TECHNICAL_OWNER, - ) - ] - ) + if self.ingest_owner and dataset.owner: + owner_urn = ( + make_user_urn(dataset.owner) + if dataset.owner_type == "USER" + else make_group_urn(dataset.owner) + ) + ownership: OwnershipClass = OwnershipClass( + owners=[ + OwnerClass( + owner=owner_urn, + type=OwnershipTypeClass.TECHNICAL_OWNER, + ) + ] + ) + return ownership + + return None def _create_glossary_terms(self, entity: DremioDataset) -> GlossaryTermsClass: return GlossaryTermsClass( diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py index d966d575c03320..b3f2107a1dfaa7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_config.py @@ -174,3 +174,8 @@ def is_profiling_enabled(self) -> bool: default=False, description="Whether to include query-based lineage information.", ) + + ingest_owner: bool = Field( + default=True, + description="Ingest Owner from source. This will override Owner info entered from UI", + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py index 5b96845ec04961..5535a406177016 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py @@ -97,6 +97,7 @@ class DremioSource(StatefulIngestionSourceBase): - Ownership and Glossary Terms: - Metadata related to ownership of datasets, extracted from Dremio’s ownership model. - Glossary terms and business metadata associated with datasets, providing additional context to the data. + - Note: Ownership information will only be available for the Cloud and Enterprise editions, it will not be available for the Community edition. - Optional SQL Profiling (if enabled): - Table, row, and column statistics can be profiled and ingested via optional SQL queries. @@ -123,6 +124,7 @@ def __init__(self, config: DremioSourceConfig, ctx: PipelineContext): self.dremio_aspects = DremioAspects( platform=self.get_platform(), domain=self.config.domain, + ingest_owner=self.config.ingest_owner, platform_instance=self.config.platform_instance, env=self.config.env, ui_url=dremio_api.ui_url, From 74a84885df604436df182b01810f287f152ccb73 Mon Sep 17 00:00:00 2001 From: Jonny Dixon <45681293+acrylJonny@users.noreply.github.com> Date: Fri, 29 Nov 2024 19:46:29 +0000 Subject: [PATCH 101/174] feat(ingest/dremio): Retrieve default_schema for SQL views (#11832) --- .../src/datahub/ingestion/source/dremio/dremio_api.py | 11 +++++++++++ .../ingestion/source/dremio/dremio_entities.py | 4 ++++ .../datahub/ingestion/source/dremio/dremio_source.py | 1 + 3 files changed, 16 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py index 7b9ccb52acbef4..7f4e0f520b7a5e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py @@ -774,3 +774,14 @@ def process_source_and_containers(source): containers.extend(future.result()) return containers + + def get_context_for_vds(self, resource_id: str) -> str: + context_array = self.get( + url=f"/catalog/{resource_id}", + ).get("sqlContext") + if context_array: + return ".".join( + f'"{part}"' if "." in part else f"{part}" for part in context_array + ) + else: + return "" diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py index 16774c2e4a816f..b80d7b8e0f9123 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_entities.py @@ -200,6 +200,7 @@ class DremioDataset: columns: List[DremioDatasetColumn] sql_definition: Optional[str] dataset_type: DremioDatasetType + default_schema: Optional[str] owner: Optional[str] owner_type: Optional[str] created: str @@ -235,6 +236,9 @@ def __init__( if self.sql_definition: self.dataset_type = DremioDatasetType.VIEW + self.default_schema = api_operations.get_context_for_vds( + resource_id=self.resource_id + ) else: self.dataset_type = DremioDatasetType.TABLE diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py index 5535a406177016..f814108c377605 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py @@ -417,6 +417,7 @@ def process_dataset( view_urn=dataset_urn, view_definition=dataset_info.sql_definition, default_db=self.default_db, + default_schema=dataset_info.default_schema, ) elif dataset_info.dataset_type == DremioDatasetType.TABLE: From 337f2b95f552c38577117ea8cdc5228c719d7001 Mon Sep 17 00:00:00 2001 From: Jonny Dixon <45681293+acrylJonny@users.noreply.github.com> Date: Fri, 29 Nov 2024 19:59:18 +0000 Subject: [PATCH 102/174] fix(docs): fix sample business glossary (#11669) --- .../datahub-business-glossary.md | 20 +++++++++---------- .../banking_business_glossary.yaml | 4 ++-- .../bootstrap_data/business_glossary.yml | 12 +++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md b/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md index 3d2a0509492bd5..3433a853ea9b05 100644 --- a/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md +++ b/metadata-ingestion/docs/sources/business-glossary/datahub-business-glossary.md @@ -7,7 +7,7 @@ The business glossary source file should be a .yml file with the following top-l Example **Glossary**: ```yaml -version: 1 # the version of business glossary file config the config conforms to. Currently the only version released is `1`. +version: "1" # the version of business glossary file config the config conforms to. Currently the only version released is `1`. source: DataHub # the source format of the terms. Currently only supports `DataHub` owners: # owners contains two nested fields users: # (optional) a list of user IDs @@ -60,7 +60,7 @@ Example **GlossaryTerm**: - Shipping.CountryCode - Shipping.StreetAddress custom_properties: # (optional) a map of key/value pairs of arbitrary custom properties - - is_used_for_compliance_tracking: true + - is_used_for_compliance_tracking: "true" knowledge_links: # (optional) a list of **KnowledgeCard** related to this term. These appear as links on the glossary node's page - url: "https://en.wikipedia.org/wiki/Address" label: Wiki link @@ -73,7 +73,7 @@ To see how these all work together, check out this comprehensive example busines Example business glossary file ```yaml -version: 1 +version: "1" source: DataHub owners: users: @@ -89,15 +89,15 @@ nodes: - name: Sensitive description: Sensitive Data custom_properties: - is_confidential: false + is_confidential: "false" - name: Confidential description: Confidential Data custom_properties: - is_confidential: true + is_confidential: "true" - name: HighlyConfidential description: Highly Confidential Data custom_properties: - is_confidential: true + is_confidential: "true" domain: Marketing - name: PersonalInformation description: All terms related to personal information @@ -148,7 +148,7 @@ nodes: related_terms: - Housing.Kitchen.Cutlery custom_properties: - - is_used_for_compliance_tracking: true + - is_used_for_compliance_tracking: "true" knowledge_links: - url: "https://en.wikipedia.org/wiki/Address" label: Wiki link @@ -237,7 +237,7 @@ Source file linked [here](https://github.com/datahub-project/datahub/blob/master ## Generating custom IDs for your terms -IDs are normally inferred from the glossary term/node's name, see the `enable_auto_id` config. But, if you need a stable +IDs are normally inferred from the glossary term/node's name, see the `enable_auto_id` config. But, if you need a stable identifier, you can generate a custom ID for your term. It should be unique across the entire Glossary. Here's an example ID: @@ -247,5 +247,5 @@ A note of caution: once you select a custom ID, it cannot be easily changed. ## Compatibility -Compatible with version 1 of business glossary format. -The source will be evolved as we publish newer versions of this format. \ No newline at end of file +Compatible with version 1 of business glossary format. +The source will be evolved as we publish newer versions of this format. diff --git a/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml b/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml index d0fea81748da57..a1adec58b7b5ea 100644 --- a/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml +++ b/metadata-ingestion/examples/bootstrap_data/banking_business_glossary.yaml @@ -1,4 +1,4 @@ -version: 1 +version: "1" source: DataHub owners: users: @@ -68,4 +68,4 @@ nodes: - name: Auto Loan description: "A type of loan used to finance the purchase of a vehicle, with the vehicle serving as collateral for the loan." - name: Interest Rate - description: "The rate at which interest is charged on a loan or paid on an investment, expressed as a percentage of the principal amount." \ No newline at end of file + description: "The rate at which interest is charged on a loan or paid on an investment, expressed as a percentage of the principal amount." diff --git a/metadata-ingestion/examples/bootstrap_data/business_glossary.yml b/metadata-ingestion/examples/bootstrap_data/business_glossary.yml index 327246863b0ab0..20d1011b966893 100644 --- a/metadata-ingestion/examples/bootstrap_data/business_glossary.yml +++ b/metadata-ingestion/examples/bootstrap_data/business_glossary.yml @@ -1,4 +1,4 @@ -version: 1 +version: "1" source: DataHub owners: users: @@ -11,20 +11,20 @@ nodes: - label: Wiki link for classification url: "https://en.wikipedia.org/wiki/Classification" custom_properties: - is_confidential: true + is_confidential: "true" terms: - name: Sensitive description: Sensitive Data custom_properties: - is_confidential: false + is_confidential: "false" - name: Confidential description: Confidential Data custom_properties: - is_confidential: true + is_confidential: "true" - name: HighlyConfidential description: Highly Confidential Data custom_properties: - is_confidential: true + is_confidential: "true" domain: Marketing - name: PersonalInformation description: All terms related to personal information @@ -72,7 +72,7 @@ nodes: - Shipping.CountryCode - Shipping.StreetAddress custom_properties: - is_used_for_compliance_tracking: true + is_used_for_compliance_tracking: "true" knowledge_links: - url: "https://en.wikipedia.org/wiki/Address" label: Wiki link From 02198f7b27515c78d9ff3dcb1fc651a6d0442017 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Sat, 30 Nov 2024 01:36:10 -0800 Subject: [PATCH 103/174] fix(java-sdk): custom properties patch client (#11984) --- .../builder/CustomPropertiesPatchBuilder.java | 13 +- .../builder/DataFlowInfoPatchBuilder.java | 25 +- .../builder/DataJobInfoPatchBuilder.java | 8 - .../DatasetPropertiesPatchBuilder.java | 8 - .../IntermediatePatchBuilder.java | 9 - .../builder/DataFlowInfoPatchBuilderTest.java | 280 ++++++++++++++++++ .../DataJobInputOutputPatchBuilderTest.java | 237 +++++++++++++++ 7 files changed, 533 insertions(+), 47 deletions(-) create mode 100644 entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilderTest.java create mode 100644 entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilderTest.java diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java index e4143851afbe51..b78d563147e636 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java @@ -17,10 +17,16 @@ public class CustomPropertiesPatchBuilder> operations = new ArrayList<>(); + private final List> operations; public CustomPropertiesPatchBuilder(T parentBuilder) { this.parent = parentBuilder; + if (parentBuilder != null) { + // If a parent builder is provided, we use the same path operations list. + this.operations = parentBuilder.getPathValues(); + } else { + this.operations = new ArrayList<>(); + } } /** @@ -72,9 +78,4 @@ public CustomPropertiesPatchBuilder setProperties(Map propert public T getParent() { return parent; } - - @Override - public List> getSubPaths() { - return operations; - } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java index 6a114d90875fe3..231956a2fcec81 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java @@ -4,12 +4,10 @@ import static com.linkedin.metadata.Constants.DATA_FLOW_ENTITY_NAME; import static com.linkedin.metadata.Constants.DATA_FLOW_INFO_ASPECT_NAME; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.TimeStamp; import com.linkedin.metadata.aspect.patch.PatchOperationType; import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport; -import java.util.List; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -87,28 +85,23 @@ public DataFlowInfoPatchBuilder setCreated(@Nullable TimeStamp created) { } public DataFlowInfoPatchBuilder setLastModified(@Nullable TimeStamp lastModified) { + ObjectNode lastModifiedNode = instance.objectNode(); if (lastModified == null) { pathValues.add( ImmutableTriple.of( PatchOperationType.REMOVE.getValue(), BASE_PATH + LAST_MODIFIED_KEY, null)); + } else { + lastModifiedNode.put(TIME_KEY, lastModified.getTime()); + if (lastModified.getActor() != null) { + lastModifiedNode.put(ACTOR_KEY, lastModified.getActor().toString()); + } + pathValues.add( + ImmutableTriple.of( + PatchOperationType.ADD.getValue(), BASE_PATH + LAST_MODIFIED_KEY, lastModifiedNode)); } - ObjectNode lastModifiedNode = instance.objectNode(); - lastModifiedNode.put(TIME_KEY, lastModified.getTime()); - if (lastModified.getActor() != null) { - lastModifiedNode.put(ACTOR_KEY, lastModified.getActor().toString()); - } - pathValues.add( - ImmutableTriple.of( - PatchOperationType.ADD.getValue(), BASE_PATH + LAST_MODIFIED_KEY, lastModifiedNode)); return this; } - @Override - protected List> getPathValues() { - pathValues.addAll(customPropertiesPatchBuilder.getSubPaths()); - return pathValues; - } - @Override protected String getAspectName() { return DATA_FLOW_INFO_ASPECT_NAME; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java index 99c0ac6c15eb1a..dd17fbacf338eb 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java @@ -4,13 +4,11 @@ import static com.linkedin.metadata.Constants.DATA_JOB_ENTITY_NAME; import static com.linkedin.metadata.Constants.DATA_JOB_INFO_ASPECT_NAME; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.common.TimeStamp; import com.linkedin.common.urn.DataFlowUrn; import com.linkedin.metadata.aspect.patch.PatchOperationType; import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport; -import java.util.List; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -113,12 +111,6 @@ public DataJobInfoPatchBuilder setLastModified(@Nullable TimeStamp lastModified) return this; } - @Override - protected List> getPathValues() { - pathValues.addAll(customPropertiesPatchBuilder.getSubPaths()); - return pathValues; - } - @Override protected String getAspectName() { return DATA_JOB_INFO_ASPECT_NAME; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java index 31e181fc244fba..60d52c7c720881 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java @@ -4,10 +4,8 @@ import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; import static com.linkedin.metadata.Constants.DATASET_PROPERTIES_ASPECT_NAME; -import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.metadata.aspect.patch.PatchOperationType; import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport; -import java.util.List; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -116,12 +114,6 @@ public DatasetPropertiesPatchBuilder setCustomProperties(Map pro return this; } - @Override - protected List> getPathValues() { - pathValues.addAll(customPropertiesPatchBuilder.getSubPaths()); - return pathValues; - } - @Override protected String getAspectName() { return DATASET_PROPERTIES_ASPECT_NAME; diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java index d891a6b9673da0..cd74818c24e191 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java @@ -1,9 +1,6 @@ package com.linkedin.metadata.aspect.patch.builder.subtypesupport; -import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.metadata.aspect.patch.builder.AbstractMultiFieldPatchBuilder; -import java.util.List; -import org.apache.commons.lang3.tuple.ImmutableTriple; /** * Used for supporting intermediate subtypes when constructing a patch for an aspect that includes @@ -15,10 +12,4 @@ public interface IntermediatePatchBuilder> getSubPaths(); } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilderTest.java new file mode 100644 index 00000000000000..612282b7c0238c --- /dev/null +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilderTest.java @@ -0,0 +1,280 @@ +package com.linkedin.metadata.aspect.patch.builder; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.linkedin.common.TimeStamp; +import com.linkedin.common.urn.Urn; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class DataFlowInfoPatchBuilderTest { + + private TestableDataFlowInfoPatchBuilder builder; + private static final String TEST_URN = "urn:li:dataFlow:(test,flow1,PROD)"; + + // Test helper class to expose protected method + private static class TestableDataFlowInfoPatchBuilder extends DataFlowInfoPatchBuilder { + public List> getTestPathValues() { + return getPathValues(); + } + } + + @BeforeMethod + public void setup() throws URISyntaxException { + builder = new TestableDataFlowInfoPatchBuilder(); + builder.urn(Urn.createFromString(TEST_URN)); + } + + @Test + public void testBuildDoesNotAffectPathValues() throws URISyntaxException { + String testName = "testFlow"; + String testDescription = "Test description"; + + builder.setName(testName).setDescription(testDescription).addCustomProperty("key1", "value1"); + + // First call build() + builder.build(); + + // Then verify we can still access pathValues and they're correct + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 3); + + // Verify the operations are still intact + assertEquals(pathValues.get(0).getLeft(), "add"); + assertEquals(pathValues.get(0).getMiddle(), "/name"); + assertEquals(pathValues.get(0).getRight().asText(), testName); + + assertEquals(pathValues.get(1).getLeft(), "add"); + assertEquals(pathValues.get(1).getMiddle(), "/description"); + assertEquals(pathValues.get(1).getRight().asText(), testDescription); + + assertEquals(pathValues.get(2).getLeft(), "add"); + assertTrue(pathValues.get(2).getMiddle().startsWith("/customProperties/")); + assertEquals(pathValues.get(2).getRight().asText(), "value1"); + + // Verify we can call build() again without issues + builder.build(); + + // And verify pathValues are still accessible and correct + pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 3); + } + + @Test + public void testSetName() { + String testName = "testFlow"; + builder.setName(testName); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertEquals(operation.getMiddle(), "/name"); + assertEquals(operation.getRight().asText(), testName); + } + + @Test + public void testSetDescription() { + String testDescription = "Test description"; + builder.setDescription(testDescription); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertEquals(operation.getMiddle(), "/description"); + assertEquals(operation.getRight().asText(), testDescription); + } + + @Test + public void testSetDescriptionNull() { + builder.setDescription(null); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertEquals(operation.getMiddle(), "/description"); + assertNull(operation.getRight()); + } + + @Test + public void testSetProject() { + String testProject = "testProject"; + builder.setProject(testProject); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertEquals(operation.getMiddle(), "/project"); + assertEquals(operation.getRight().asText(), testProject); + } + + @Test + public void testSetProjectNull() { + builder.setProject(null); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertEquals(operation.getMiddle(), "/project"); + assertNull(operation.getRight()); + } + + @Test + public void testSetCreated() throws URISyntaxException { + long time = System.currentTimeMillis(); + String actor = "urn:li:corpuser:testUser"; + TimeStamp created = new TimeStamp(); + created.setTime(time); + created.setActor(Urn.createFromString(actor)); + + builder.setCreated(created); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertEquals(operation.getMiddle(), "/created"); + JsonNode createdNode = operation.getRight(); + assertTrue(createdNode.isObject()); + assertEquals(createdNode.get("time").asLong(), time); + assertEquals(createdNode.get("actor").asText(), actor); + } + + @Test + public void testSetCreatedNull() { + builder.setCreated(null); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertEquals(operation.getMiddle(), "/created"); + assertNull(operation.getRight()); + } + + @Test + public void testSetLastModified() throws URISyntaxException { + long time = System.currentTimeMillis(); + String actor = "urn:li:corpuser:testUser"; + TimeStamp lastModified = new TimeStamp(); + lastModified.setTime(time); + lastModified.setActor(Urn.createFromString(actor)); + + builder.setLastModified(lastModified); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertEquals(operation.getMiddle(), "/lastModified"); + JsonNode lastModifiedNode = operation.getRight(); + assertTrue(lastModifiedNode.isObject()); + assertEquals(lastModifiedNode.get("time").asLong(), time); + assertEquals(lastModifiedNode.get("actor").asText(), actor); + } + + @Test + public void testSetLastModifiedNull() { + builder.setLastModified(null); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertEquals(operation.getMiddle(), "/lastModified"); + assertNull(operation.getRight()); + } + + @Test + public void testAddCustomProperties() { + builder.addCustomProperty("key1", "value1").addCustomProperty("key2", "value2"); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 2); + + pathValues.forEach( + operation -> { + assertEquals(operation.getLeft(), "add"); + assertTrue(operation.getMiddle().startsWith("/customProperties/")); + assertTrue(operation.getRight().isTextual()); + }); + } + + @Test + public void testRemoveCustomProperty() { + builder.removeCustomProperty("key1"); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertEquals(operation.getMiddle(), "/customProperties/key1"); + assertNull(operation.getRight()); + } + + @Test + public void testSetCustomProperties() { + Map properties = new HashMap<>(); + properties.put("key1", "value1"); + properties.put("key2", "value2"); + + builder.setCustomProperties(properties); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertEquals(operation.getMiddle(), "/customProperties"); + assertTrue(operation.getRight().isObject()); + } +} diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilderTest.java new file mode 100644 index 00000000000000..dc141863e24438 --- /dev/null +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/builder/DataJobInputOutputPatchBuilderTest.java @@ -0,0 +1,237 @@ +package com.linkedin.metadata.aspect.patch.builder; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.linkedin.common.Edge; +import com.linkedin.common.urn.DataJobUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.graph.LineageDirection; +import java.net.URISyntaxException; +import java.util.List; +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class DataJobInputOutputPatchBuilderTest { + + private TestableDataJobInputOutputPatchBuilder builder; + private static final String TEST_DATAJOB_URN = + "urn:li:dataJob:(urn:li:dataFlow:(test,flow1,PROD),job1)"; + private static final String TEST_DATASET_URN = + "urn:li:dataset:(urn:li:dataPlatform:hive,SampleTable,PROD)"; + private static final String TEST_DATASET_FIELD_URN = + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,SampleTable,PROD),id)"; + + // Test helper class to expose protected method + private static class TestableDataJobInputOutputPatchBuilder + extends DataJobInputOutputPatchBuilder { + public List> getTestPathValues() { + return getPathValues(); + } + } + + @BeforeMethod + public void setup() throws URISyntaxException { + builder = new TestableDataJobInputOutputPatchBuilder(); + builder.urn(Urn.createFromString(TEST_DATAJOB_URN)); + } + + @Test + public void testBuildDoesNotAffectPathValues() throws URISyntaxException { + DataJobUrn dataJobUrn = DataJobUrn.createFromString(TEST_DATAJOB_URN); + DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN); + + builder + .addInputDatajobEdge(dataJobUrn) + .addInputDatasetEdge(datasetUrn) + .addOutputDatasetEdge(datasetUrn); + + // First call build() + builder.build(); + + // Then verify we can still access pathValues and they're correct + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 3); + + // Verify we can call build() again without issues + builder.build(); + + // And verify pathValues are still accessible and correct + pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 3); + } + + @Test + public void testAddInputDatajobEdge() throws URISyntaxException { + DataJobUrn dataJobUrn = DataJobUrn.createFromString(TEST_DATAJOB_URN); + builder.addInputDatajobEdge(dataJobUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertTrue(operation.getMiddle().startsWith("/inputDatajobEdges/")); + assertTrue(operation.getRight().isObject()); + assertEquals(operation.getRight().get("destinationUrn").asText(), dataJobUrn.toString()); + } + + @Test + public void testRemoveInputDatajobEdge() throws URISyntaxException { + DataJobUrn dataJobUrn = DataJobUrn.createFromString(TEST_DATAJOB_URN); + builder.removeInputDatajobEdge(dataJobUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertTrue(operation.getMiddle().startsWith("/inputDatajobEdges/")); + assertNull(operation.getRight()); + } + + @Test + public void testAddInputDatasetEdge() throws URISyntaxException { + DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN); + builder.addInputDatasetEdge(datasetUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertTrue(operation.getMiddle().startsWith("/inputDatasetEdges/")); + assertTrue(operation.getRight().isObject()); + assertEquals(operation.getRight().get("destinationUrn").asText(), datasetUrn.toString()); + } + + @Test + public void testRemoveInputDatasetEdge() throws URISyntaxException { + DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN); + builder.removeInputDatasetEdge(datasetUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertTrue(operation.getMiddle().startsWith("/inputDatasetEdges/")); + assertNull(operation.getRight()); + } + + @Test + public void testAddOutputDatasetEdge() throws URISyntaxException { + DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN); + builder.addOutputDatasetEdge(datasetUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertTrue(operation.getMiddle().startsWith("/outputDatasetEdges/")); + assertTrue(operation.getRight().isObject()); + assertEquals(operation.getRight().get("destinationUrn").asText(), datasetUrn.toString()); + } + + @Test + public void testAddInputDatasetField() throws URISyntaxException { + Urn fieldUrn = Urn.createFromString(TEST_DATASET_FIELD_URN); + builder.addInputDatasetField(fieldUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertTrue(operation.getMiddle().startsWith("/inputDatasetFields/")); + assertTrue(operation.getRight().isTextual()); + assertEquals(operation.getRight().asText(), fieldUrn.toString()); + } + + @Test + public void testRemoveInputDatasetField() throws URISyntaxException { + Urn fieldUrn = Urn.createFromString(TEST_DATASET_FIELD_URN); + builder.removeInputDatasetField(fieldUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "remove"); + assertTrue(operation.getMiddle().startsWith("/inputDatasetFields/")); + assertNull(operation.getRight()); + } + + @Test + public void testAddOutputDatasetField() throws URISyntaxException { + Urn fieldUrn = Urn.createFromString(TEST_DATASET_FIELD_URN); + builder.addOutputDatasetField(fieldUrn); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertTrue(operation.getMiddle().startsWith("/outputDatasetFields/")); + assertTrue(operation.getRight().isTextual()); + assertEquals(operation.getRight().asText(), fieldUrn.toString()); + } + + @Test + public void testAddEdgeWithDirection() throws URISyntaxException { + DatasetUrn datasetUrn = DatasetUrn.createFromString(TEST_DATASET_URN); + Edge edge = new Edge(); + edge.setDestinationUrn(datasetUrn); + + builder.addEdge(edge, LineageDirection.UPSTREAM); + builder.build(); + + List> pathValues = builder.getTestPathValues(); + assertNotNull(pathValues); + assertEquals(pathValues.size(), 1); + + ImmutableTriple operation = pathValues.get(0); + assertEquals(operation.getLeft(), "add"); + assertTrue(operation.getMiddle().startsWith("/inputDatasetEdges/")); + assertTrue(operation.getRight().isObject()); + assertEquals(operation.getRight().get("destinationUrn").asText(), datasetUrn.toString()); + } + + @Test + public void testInvalidEntityTypeThrowsException() throws URISyntaxException { + Urn invalidUrn = Urn.createFromString("urn:li:glossaryTerm:invalid"); + Edge edge = new Edge(); + edge.setDestinationUrn(invalidUrn); + + assertThrows( + IllegalArgumentException.class, + () -> { + builder.addEdge(edge, LineageDirection.UPSTREAM); + }); + } +} From a31c88e622cf960ee99182553c10f21b525eea67 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Mon, 2 Dec 2024 10:03:06 +0100 Subject: [PATCH 104/174] fix[ingest/build]: Disable preflight script as it is not needed anymore (#11989) --- metadata-ingestion/build.gradle | 10 +- .../scripts/datahub_preflight.sh | 108 ------------------ 2 files changed, 1 insertion(+), 117 deletions(-) delete mode 100755 metadata-ingestion/scripts/datahub_preflight.sh diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index 4e3f1ca91766c2..4e03dd6e2faaf2 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -30,15 +30,7 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { "touch ${sentinel_file}" } -task runPreFlightScript(type: Exec, dependsOn: environmentSetup) { - def sentinel_file = ".preflight_sentinel" - outputs.file(sentinel_file) - commandLine 'bash', '-c', - "scripts/datahub_preflight.sh && " + - "touch ${sentinel_file}" -} - -task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) { +task installPackageOnly(type: Exec, dependsOn: environmentSetup) { def sentinel_file = "${venv_name}/.build_install_package_only_sentinel" inputs.file file('setup.py') outputs.file(sentinel_file) diff --git a/metadata-ingestion/scripts/datahub_preflight.sh b/metadata-ingestion/scripts/datahub_preflight.sh deleted file mode 100755 index 9676964f4d49d1..00000000000000 --- a/metadata-ingestion/scripts/datahub_preflight.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash -e - -#From https://stackoverflow.com/questions/4023830/how-to-compare-two-strings-in-dot-separated-version-format-in-bash -verlte() { - [ "$1" == "$(echo -e "$1\n$2" | sort -V | head -n1)" ] -} - -brew_install() { - package=${1} - required_version=${2} - printf '\n🔎 Checking if %s installed\n' "${package}" - version=$(brew list --version|grep "$1"|awk '{ print $2 }') - - if [ -n "${version}" ]; then - if [ -n "$2" ] && ! verlte "${required_version}" "${version}"; then - printf '🔽 %s is installed but its version %s is lower than the required %s\n' "${package}" "${version}" "${required_version}. Updating version..." - brew update && brew upgrade "$1" && printf '✅ %s is installed\n' "${package}" - else - printf '✅ %s is already installed\n' "${package} with version ${version}" - fi - else - brew install "$1" && printf '✅ %s is installed\n' "${package}" - fi -} - -arm64_darwin_preflight() { - printf "✨ Creating/activating Virtual Environment\n" - python3 -m venv venv - source venv/bin/activate - - printf "🔎 Checking if Scipy installed\n" - if pip list | grep -F scipy; then - printf "✅ Scipy already installed\n" - else - printf "Scipy not installed\n" - printf "⛅ Installing prerequisities for scipy" - brew install openblas - OPENBLAS="$(brew --prefix openblas)" - export OPENBLAS - ##preinstall numpy and pythran from source - pip3 uninstall -y numpy pythran - pip3 install cython pybind11 - pip3 install --no-use-pep517 numpy - pip3 install pythran - pip3 install --no-use-pep517 scipy - fi - - brew_install "openssl@1.1" - brew install "postgresql@14" - - # postgresql installs libs in a strange way - # we first symlink /opt/postgresql@14 to /opt/postgresql - if [ ! -z $(brew --prefix)/opt/postgresql ]; then - printf "✨ Symlinking postgresql@14 to postgresql\n" - ln -sf $(brew --prefix postgresql@14) $(brew --prefix)/opt/postgresql - fi - # we then symlink all libs under /opt/postgresql@14/lib/postgresql@14 to /opt/postgresql@14/lib - if [ ! -z $(brew --prefix postgresql@14)/lib/postgresql@14 ]; then - printf "✨ Patching up libs in $(brew --prefix postgresql@14)/lib/postgresql@14)\n" - ln -sf $(brew --prefix postgresql@14)/lib/postgresql@14/* $(brew --prefix postgresql@14)/lib/ - fi - - printf "\e[38;2;0;255;0m✅ Done\e[38;2;255;255;255m\n" - - printf "✨ Setting up environment variable:\n" - GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 - export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL - GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 - export GRPC_PYTHON_BUILD_SYSTEM_ZLIB - CPPFLAGS="-I$(brew --prefix openssl@1.1)/include" - export CPPFLAGS - LDFLAGS="-L$(brew --prefix openssl@1.1)/lib" - export LDFLAGS - -cat << EOF - export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 - export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 - export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include" - export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib -L$(brew --prefix postgresql@14)/lib/postgresql@14" - -EOF - - if pip list | grep -F confluent-kafka; then - printf "✅ confluent-kafka already installed\n" - else - pip3 install confluent-kafka - fi - - printf "✨ Setting up prerequisities\n" - # none for now, since jq was removed - - printf "\e[38;2;0;255;0m✅ Done\e[38;2;255;255;255m\n" -} - - -printf "🔎 Checking if current directory is metadata-ingestion folder\n" -if [ "$(basename "$(pwd)")" != "metadata-ingestion" ]; then - printf "💥 You should run this script in Datahub\'s metadata-ingestion folder but your folder is %s\n" "$(pwd)" - exit 123 -fi -printf '✅ Current folder is metadata-ingestion (%s) folder\n' "$(pwd)" -if [[ $(uname -m) == 'arm64' && $(uname) == 'Darwin' ]]; then - printf "👟 Running preflight for m1 mac\n" - arm64_darwin_preflight -fi - - -printf "\n\e[38;2;0;255;0m✅ Preflight was successful\e[38;2;255;255;255m\n" From dc87b51369030ace07f48d857dc02c1c64bc911d Mon Sep 17 00:00:00 2001 From: k-bartlett Date: Mon, 2 Dec 2024 04:23:28 -0500 Subject: [PATCH 105/174] feat(ingest): connector for Neo4j (#11526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: kbartlett Co-authored-by: Andrew Sikowitz Co-authored-by: Jay Feldman <8128360+feldjay@users.noreply.github.com> Co-authored-by: Harshal Sheth Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Co-authored-by: Shirshanka Das Co-authored-by: deepgarg-visa <149145061+deepgarg-visa@users.noreply.github.com> Co-authored-by: Felix Lüdin <13187726+Masterchen09@users.noreply.github.com> --- .../app/ingest/source/builder/constants.ts | 4 + .../app/ingest/source/builder/sources.json | 8 + datahub-web-react/src/images/neo4j.png | Bin 0 -> 12968 bytes .../docs/sources/neo4j/neo4j.md | 20 ++ .../docs/sources/neo4j/neo4j_recipe.yml | 12 + metadata-ingestion/setup.py | 3 + .../ingestion/source/common/subtypes.py | 2 + .../ingestion/source/neo4j/__init__.py | 0 .../ingestion/source/neo4j/neo4j_source.py | 331 ++++++++++++++++++ .../tests/unit/test_neo4j_source.py | 221 ++++++++++++ .../bootstrap_mcps/data-platforms.yaml | 11 + 11 files changed, 612 insertions(+) create mode 100644 datahub-web-react/src/images/neo4j.png create mode 100644 metadata-ingestion/docs/sources/neo4j/neo4j.md create mode 100644 metadata-ingestion/docs/sources/neo4j/neo4j_recipe.yml create mode 100644 metadata-ingestion/src/datahub/ingestion/source/neo4j/__init__.py create mode 100644 metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py create mode 100644 metadata-ingestion/tests/unit/test_neo4j_source.py diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index f892f0ed525d25..58525b3e88f975 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -38,6 +38,7 @@ import sigmaLogo from '../../../../images/sigmalogo.png'; import sacLogo from '../../../../images/saclogo.svg'; import cassandraLogo from '../../../../images/cassandralogo.png'; import datahubLogo from '../../../../images/datahublogo.png'; +import neo4j from '../../../../images/neo4j.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -137,6 +138,8 @@ export const DATAHUB_GC = 'datahub-gc'; export const DATAHUB_LINEAGE_FILE = 'datahub-lineage-file'; export const DATAHUB_BUSINESS_GLOSSARY = 'datahub-business-glossary'; export const DATAHUB_URN = `urn:li:dataPlatform:${DATAHUB}`; +export const NEO4J = 'neo4j'; +export const NEO4J_URN = `urn:li:dataPlatform:${NEO4J}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -180,6 +183,7 @@ export const PLATFORM_URN_TO_LOGO = { [SAC_URN]: sacLogo, [CASSANDRA_URN]: cassandraLogo, [DATAHUB_URN]: datahubLogo, + [NEO4J_URN]: neo4j, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 44b8a37f14655d..776b6703895c35 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -325,5 +325,13 @@ "description": "Ingest databases and tables from any Iceberg catalog implementation", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/iceberg", "recipe": "source:\n type: \"iceberg\"\n config:\n env: dev\n # each thread will open internet connections to fetch manifest files independently, \n # this value needs to be adjusted with ulimit\n processing_threads: 1 \n # a single catalog definition with a form of a dictionary\n catalog: \n demo: # name of the catalog\n type: \"rest\" # other types are available\n uri: \"uri\"\n s3.access-key-id: \"access-key\"\n s3.secret-access-key: \"secret-access-key\"\n s3.region: \"aws-region\"\n profiling:\n enabled: false\n" + }, + { + "urn": "urn:li:dataPlatform:neo4j", + "name": "neo4j", + "displayName": "Neo4j", + "description": "Import Nodes and Relationships from Neo4j.", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/neo4j/", + "recipe": "source:\n type: 'neo4j'\n config:\n uri: 'neo4j+ssc://host:7687'\n username: 'neo4j'\n password: 'password'\n env: 'PROD'\n\nsink:\n type: \"datahub-rest\"\n config:\n server: 'http://localhost:8080'" } ] diff --git a/datahub-web-react/src/images/neo4j.png b/datahub-web-react/src/images/neo4j.png new file mode 100644 index 0000000000000000000000000000000000000000..b03b2a4532b3ba0329fdaccbb60a3a57b111ed4f GIT binary patch literal 12968 zcmW+-1y~#177XqV1&TYxrMP>s;!xb7xNC5Cid%7aEACzhUfiWXad&(Bzx@(6Azyay z-h0lRnK_$C<*zbm$RCkGAP}1D7fDrM{{%eM5TSv;eA%%Izz)hqRYn|Yd6dZ?cmr=D zFCz);f$dSE1NaXzt(Am?vX!YF2;})QJJmy7eepy1RF{W${sfyVfp@JcJmw)uk(LU9 z1w6qRXaIEyg^~eNMY8Bq5G))j0h*MF6fO#s3KUp0C;&TH#kR~$dO^;|%jng&L}%*K}mZl5v(uOi3=kBbcqu89t8Ju9{RFa?+@p~MO&mS zbUUehuD!htBoFN5tjY%+=tQ2B(*L*du;$Q;P+{XE=)?JoHpkA&tdvwUcS}<kpE#+%N z_OK)T`LdH536{{kq=b);k5&&IBB?jMc`HKl7S`>V8~M<>36b#FrDh8)hi-4RuezCY zrO3@m=`(ZsH|$Tk<4^ps=tThMg3arLu7r!llF7lssxoI!n#H(Pa{W(l!{2&G1ob)T z!+{R9+$)`EEEkk2jC#P!8iFz|rKkydlzZOh^L{u6LSO-xxSg}pnFz|$z1p_PB~^6R z1K%VUZ}KGeRU`iWU@na)4D=6J^*oSzAni2Fa4+-{2)qTm7Knie$|pyzf@b;wI}0NF z8*poc@*5%4h-V3bK@6=7AvO>0Cx~S0b1R4?_;?GR0Q$agG%jAi+Pl4DkZWaOk zN!0oWw;7_gDC&2@zvz!*eag7F2sGk?Ul6Oqo5e$wk<^jq#bc*ira%3IcE$S`awcv% z{rQIRhE&=JTOm}#h-3-ZqIa^6l^D4;1ZP{plldpYc@Siq|AAl)-7uhbK;nY_1vN(k zBrf?qg5M;D4O5F66e*33DcpedNp$Q7e-YfFxOal?Z+ed4P6=b+-#*73)n`}-O?$?p%fVR1uZZ+<1z+J>XE-mRKakAAcM$tk4wlW>?eFDiYKsl z%yAKF1KL}Hjx)TZEcjYsHvQQ{>w}6r`a>Z#H zUQBVM(GcbkXK|Em>}{xRnuITNP$S;yE=I71}a_3YPh-OaB39FwLDLvHAVmlK%lLTeDMY}cpzPxoo>-XtXNEyeA zhxbcJB^Lym{4j~z5gU?^;*?XT0n?p8r1YStdJe_MP^7s7YcXWTphH09F*q(9S5%oX=Y4Nn^if zpW-^_c%Sd@4Ki0TcLF1MMUEsY%JZjn)_wl<_Z#W2PE)D+tCkh}dAr-0mElZ>brT`G zLUSBTH9OA!JPv3=sTjuQ@4EP_ip`~Vo6Fj(NGD&HUQHZqb4@nQM4Hq+K3Q75&N?BW zA(Ti|5B%eJYG^eNPmSfT5a1v=CMFZG;;+yC_>(sKHajUho=?Ps-DSJuS6io3lgs*+ z_Xe8NwJW3Ju}7K{hD)uh|KI%Wgnu(9?8jDnZIjoX_$!b_oA||umM9)b#|(I3WY{uT zDV}wdXRu}VzCJgQY_k?Zc6WcddHinFcIUit6$irHBB|rqGS0I%ve>#8OV8`gtH`74 z+wVIL220RW)Ke%@)Jo`!M@yWCABG|gv=7h>nD!G5Y)ZCDB}>LgXG+^gbdnYm48X0@JLQoI~vwI3`xdlDEN(E$gQVTY8mJkw=%fRXM7v|W^ppg>#8TIr?!S*0cwon z!#^i6BoTo)o1a<=nF`d)FaIG%_=;r7(MOR_ZOkx3;pDL6nrN!3J&U@SqC4uFaFU{z z^g>HRzV~HJ+*;Dw<|7L;%PFo>SX%$Ftd^o-ws5*}dO|LHrd3)St4r!R#Nl^PgQkTW za#=xcxp01tW0oTmSleCWZuRHI&!oy?t!vHuYW8o9E)#!s;+qU>C8|xUj}2h_O@ zrP&m2F3rygfeuIe)3&y2!UIbdY6F7vxYKTb_N$%KoV=Vj`R=qDy}MSQ`o*^UHN0@f z(%Kb$A-7nk>fQF++dJXP?q;$ozwgH_XAWd_g#S8?!G$)Chg-{8aXhSR&K+)#&G)Cr z5|D8C@PV-Nn8BEs4dcy>_AKxFp{^6~^*xV>s0h6${cYVz!c}2A*{#1VNfW6M=RWts zJ@4PkSEq%C#@q(KC$@z9ke4D`Rl5eKF_WX4Un4iMrHnaW1tFf?B1!L!H&gRdw>kdz z(SKNd8xtCFSRYtF_%VBzUtIlWglMh11iHiF#t6gYrpU!y1cyygpZ75A1`1JbG zzn8t&A19rcBA{5Ip!7i9hg?6?uS__DK=h=tl49zfE2q9*KDa~8@8xGXquq_L*r}N0 z4fOPqEU=W%lo0_}(e%y<6C)a41wmSHDd8wl5yE7iT})bVF#?WKtn_1)-w`oUqp6u> z1FERasFKVq+AID1e=5vmoE=gp4Q*LYhGXOwsp-2_9O;&R&2~BZY#b99F6fGbZ5o3q zPG}kv8I;GIN3V{FNC7Jk9dILaeL{iJCc*WsQ~?ta)A_`9cry4?IpMPT{DQJ!E|d_z zQk#v@=D#nXyci_(L7aHk`)NfP>#8)p7AAv^+uikW?ItuyC`71X(R>S$T$$21&+ALV z4U0c$gW#>R{Hz3ebC9y00lV?#ktyA5$TYbac`pGBIY>03Kv`3d`=};RVAFGU6(Wg< z1qvDFKBmBEVq&K@3hC|S;U;@#USDHLW-s|2B~GYOCZ0jb2r{xKyW0&kReC$8D;32~T zfgZl!Qvig%X3lAt01>w8t{RNa3EB_r2)CEUmNH38!NIQykS#!`DojPBh~Z$9mHER8 zWg0~8iJCzA1oL}TbO*V%os|P*q^zRtwyp|S0CHQ=fmPL0DOYBQEC2>%O`$5$x&I4E z^b50%=Xxt$2%9wGUM5cJNM;ITedmzssnzr`98ZiC~4&{1qK&0pWZ7j z%~QMEv4QXHd}pVi)*krw=yr^;Rw0W!R&}iJP|lVyIV$K!Vh-0;{o+`SRwnc-eQ^v2 z2m~F1DTaAcyhz3Wi|^tMWu3oy;CA8Tw!wkH9fEV{T&9-|&52VLOu&Ri7N_M`6B9~M zM8R+3!M35>fTQQeDd7(5c?OnUGzOVLGNCN?`xLa&NsEAGSXv}RsIgv_V0}`b5$3Ek z5pVf4^wz&`59t|U%^M~-<>uo{-qZbvHhU*dshCZ`g|D``lv2xjes$9zy{x7mv~^W{#iM%nN{1OwK=(sR0u!ulWmSVvX$ZOcf^;C zGVgp-hk0Mq7!|p;I(&8ZiHXnt*2Bsi#vLpB^u1B0Ni153)kQq(luhPOR5t}pjE?eoUHx_Fc}>m9$w^95)6>&a zQK5h|kRxtvY>0`8ovyaTM@1b0vHJe@3cf$Hj@R-Y0R6SK6ml$eM;Euin1U#XAOD!5Rs zd*9OYs+el7g1c@pe^|GQ?dZ|a2`y@Ge|859*J0D|=HJZK)zy%D+OLLy`_o*ohs(s9;aiL}gVM6fP>d3h-lM~npv zWm)k?9YtCESy3rv9ckl4M zJ({;$`L)f)!4^0lUQZzrMl9(0gD*r+f?{tlXWrWPv}sXXTs+fyEew^=d8-$yWtlU_ z=C2ndmAU=fNO;JP^X})kp7b9-wkjOu_BY%wdtobn3 z0A!!jadiuSppuZG=}4RgE*u@oyJ4dRTI-K9w4e^}2p!zB&iDUEi~V{#MUkE&=&X4~ zR?66_&SN+FYXna^caw9Q*R@5}a&LnT5c|mWJwVVxCMRp?isnvyFZvLylRx8teV=Yd z$H#4{cjj%UGPzLnDo?PmumBfEFS1|j4b^d^$-e%ZiK!`CT@4onO3>EcUNKiY(Kx^6 z+jcnFN2M&CwwJ$2N}{_#>=vk?&d$zhY#mEkx+3d8=kvjM+n)Eg=Pey`d|2Q>ep{iB$X?H@ie#X1VhdXp+%y(_AevNe zk`5;cGs_AJ3uR&m*uPKbOxk=Noq^Ts3r6sN+{4eBDDjh0{Ab|x zayf#4_7N8cr|>w+6dk9QTDIrq3PKex7*tqTt@zq0pT+{@smF%b0dvdRY>mDED=RCg ztfFFUYD!67US3X)A3%w(U(tC_@~fIP@b#I?%gZr@Zl-xQeFpRp1k0D7FGt7@rgCd_ z^KroX9qt;9mfTr{rWzU=ZEbCqQ#nFD4;R}5;V395`msa}X*oHu)Kt}$9j*&Q$579%d{Xi@V zT@Ku%>hLQbkx&p=KOG!!Tx4SxNVEFUeueFj0xF@Hj5pim&s$Kz+g+V5~ zcId{HkLi0kgg+mosEgN-mX)=dFM+UY58eB4x$AMZ=JopIW_nj+(A91{5cc-^yg0gn zqa@gOl#gZ|%GjW@L3o22n06)B%m{(FB_ZN<%&W}r3sa78Hto!0)C;LemVc|$ZLtRc`rw?B z=xfYnc;`1DGR(}1#3l;pY3b-VQ?xbHx}MKErqa94);rFZ>I=nAmTj9oHhp=>$$LHV zPL~^H(u8)~hVyf{?J`+bm-GPM{9k09?ni4Shq4 z%_58G9j#)fy1M#Yp3^RTB@W%!Jm2y8>D#k)A{d7ws;zm3=PZYAtIkOPmPS60REQw0 z)oXn>U(EIt!vyegh@erY`j$P_#ofuZ#;_-@qw!3KGK89jrhX;W^QWhj%FfOX>OOcZ z`P16vAeNe?e}W^IriKOpT$6c`wLNc{^BX>M$B%cb4hV}+AZbMIWT6+bWgv~u{*=ax z;Q%ou{O2ULreY{^qs1Y&@(N{7~O91n)W$Hx_7WM{LlEOWfV*a0?i#AkWb zy$#A1^qK|0Fa_WAa%ZTjxf0L+?d|y-Er8dk7laD!GC*CL^ENv>+wJ>oP1J7~j!L-m z>(TKJU>U}}26}ow=dGpmi^Wd(kfKrWS$^5AYT8`)Y5te28XD$;ta|39$@_bI`7oRF zdI~v$8Ew4SFN9lhB+myT??@~!W(f%i=H}*Q-?VK))KLtP$7`D-MKEnvFr31h)+JmJLk&uzwJugjmmn8__Zx;=Tsw#nc zf9+M}QlL`rTHNV&y3$lvH!W-KQ4z8}=Q#4%%l5exeXF*%^9q^sv+~?BEV`kE1?}*f zwQOP$r{&~@aew{fCb;T|66} zg|bV#h*^SO4UBgHotQ+$8fMh)_Tg*gCWT$I0=0(r=l&sz^sCAbG)-`xp& zoDomfga4GIF5GW`YZ;+>>{hv10x#a(y`NpulUU+(e0mYZVRV~m_Asm?LcbjX4oGdqb;m?JT76WHHFE7pM&myb} ztf{mpE*Yl`Gj-h?OR1<}&>y`_>v@4A#-AkOgnh8LwPljNv_Z)$i5`+vNjB%Bp@IAC zVSTYk$Y}uz7&XJrjF@%`p673h`2Iy!cH0wMYR0;$q@*Nhcqn>f=aX6q+elNS^iCKU zfS&tJf05R2@1vumc^>bGUd^kgz(O-IFw7db4j)Fu#ck9X!_-VIz_=Ebpi=fe{+^uN zFWgnn^Els7oW+zI1n6^UXei!?4;8EZWB{~y6<~{D0#ueoza2L7lcZ<$xB8t4o=sbE zijdeHJE{tggm4F8pTQcvww*yz4l+EvJ%-*o_Cf^EKp2{fTj2sx7Z7Ee-GTSHUa>Akw0GrcnN*J~# z9tp{MzC`%RiEVY=@9}YUgei7l6adXR$@bWo(Wm=;vC?D%6w5&vQysj8KBTkG)v{$6 zC?ilvgy3@`#q&4mdvqFPKEIT5*)N!|cA>w6hfEU8Ge!gfc@YqehLEZ5cyy-&F%T1j z+KE&hTmGXl-tKW;qftmCp+|Rle}4{K2FqHleut*MK9E8aN>ucVe`aStFnu%1YoMd1 zX43gphF5X&liE!LN;F>q8&1X50vCq}b7J1UG#@tL=*Zf!o&{BWgGlao9rs34n69>_ z%tlH|N}x^aA9w>|d%=8>=w01=bY66$4)@c{U?^K5CMMJyi;9MZS(AECFRy&fuw6B6 z?Z>TPjMTI=8&lGHDt2~uGM@`5t>rAoNNx&QWdVm`YUEUgk3ob0l$1@$^rq$&gAUvc-oC`Ez6AEKeEavHq*9i*lvSzGhEJ-$YN z^ZbPP8IzH#NsqUB)d0TXmhyG@o2RFzvvWgyyi#>kbbXmRs0m;D09CsO6TYKJYw+amd^Qgt6(*~PNZ=Oi)^RPZOZ;q$> z{Ric-&wVi9$R$atlsDL~xm~>Hc&Do%S9h5HbP@jooExAtxmj72T1EdhGuva%d zood+AI~17908+BS#Q}Hj>@i5oDemxa{+A<8}0feXZi28DTYf z7C7*F8CS9aAkX*mbtV)Q@K}eGLX3@#g}`^Pu~cf9Aq@>2F^skhV0`XCug>0~?!Xp> zAES^oRxK?p8&fp;Vi*MUpCbOkJVT-|RB@g>v4omwuHK7i*sy_1=52XJ#FBljq+ ziU%zCW_x(^7slzC2DnnPiAM^avuhz2{a}Um?hV%NhoI6F8+&NHtse;LFl?D$c&5Jt zitc4y+Io0Afv>HzleloBNrk_F9{uCz2(U)AW@Y8d1Jdg07uC0dzO_$HNG-H2VytY# zu#^!6pt#xx!kAR*$P}8rO}bp^TWS2khiQw(?Rg^7_05M1sC&fKp$Ey*grWLpfD8O5 z1J)2eBAGkVH^IB0Hjx-SG_rudzia?3*4moEqpv;Iv*bzjzf1z^2gK8i!Y$&)eu zT0Ld$ynx;~v%{_e)K$;h({X#L5-45Ji(?;dC!Y(*pqXJ2j$t$HuV21=>+$pb@+ATs zvt9mQ{m#=q`nDZ~#|Vg;-ku?Hh%;b@B+|&Qk%oWdc`j;w>$oEg^!8McGR)mAKWIJ? z7JNQ4=I*fNUjLF1%;*3>fXn_kkd|prLt|rOMMXtGajaiqUtpGqDgf|x;wPWWGyI?xs~12FWo2idQVTp{4h;<%zfn0xxJ#)(LqlU@V=q)` z@8a?WpZEgOFN?<^hto1{!a}WQm3*U)B5CvnU|$8_7th=*8Xxn+Nq2VCRFdoJ>K+~* z0A9SJQFR*#gOF(!=Zf`m?*G1eIC|Q$+3k9$x@NFu1@E!c5H(hIs5Gz@UaWGxUBhSg zu%`2VmMKq+5sGykV%#V(EbPrdmBnt+w3^NGu@JLi`4q4N=dA&@u9+2UD4ieM9Y3J% z(F=GvNKI3^yvW%C9yL`}W);TT$suT}Xi8OkUaH`mK`f3dJcn^zQQW!HtF zFr}cDP3RrQ_Z#Gng_LUfoc9ZsFGL}8wMvXc8J48eYAx9>Rze z0C$dBum}%o2Gnh%#YF$B{;HgY#(dZP#xuZqvnDKbo|BH+Ps-mzG9yRLbbwL}M40Q| zXLJbg@p;jNnp`-bM!|QsYlOehtdx?_XkY}(7!MA5hkt_LhSTX0qr_!~?jB9%{lgzU zm!lzNKD+%`DR^z2;w)WXD=XmS;D-Mqqf|Fr=uZ;8ZaN6wza3bmIg`kXW5NZhx}V9t ziFll3-HMRK=yg7w72Bk}oYRSr2vEr;(j&|5PU@avxk%nI>$iWVq!br0`<7Q%=LndC z0R0-E!rGMRZHh#Lkf$dmCCS8oOi4`*^E9b|VFawzJ@8qZB_QRajs*T~O(|u_!7kN-Fw;(CwVspcE6R*md$J9{{OE$x(5* zw|d21=Uw+<=p@!Ht6AKfo)EQI#c*D6B$j)lp9kJ+|ZtEDXWj$*3#%7{6O~e>0V)ds)GIjpYalGc@0g2na7{dHrj9jRGrfK zrWe;{;`L|HE)x}V1-3ng&;QTxw2fpC)w&SN^gFTv!B2>x8Jw1`*C{fKra ze$Qy<7-gq9F9EumnsN}W zo}VPBcia|t#PXT4TruNN%-}EwC|vrRUK-G|@qgG3r_1pS^5>BCfHIZH;@)0mI+?IDMMBdu{UcJW` zHm^cVqmNow+riAWTokamg@idt;-5@oK;`x0oS!L>LVm2v(v`lx>I+Z|ZWo)xsGJ2l zGkpU6P)O`r!(MONorz_Idweu8KSO#5lk;dQ$4wh6l1=h^*hbLGXq|5W$W;%(1OPu9 z1p{M5feY}u0ZEg_YS`_0Nm-KQ?k%ObGzgf*ZyVrq<8kuVGp@USlX}2o+X0LrH(F!_ zkcdbRsgSqh9epu?1Awsw7+yWVcQt%}Dr-fU;Mgow^$kL`*MsziO*KA5?*TOB70`OB zYHHV4R~OvztCEO)wG02=G)UaTSR8K;9`d$l*O$|Et|ems;06Q)ifytmS|r_Uq=E6P zSD4@*HzNw?sD{%=sh6YW4E;=oHP+&~ z+rSwi5rkoz0E~TLiOL2r7fTG9_J3q$0Y?22Aoo5e3I%igZY!Xl5_x~AtnIe$#PvGI z9id~rJOJ7P!w;%v+ILr1Iz4``(X8ENjI7PJ%N&kdqFEDRt)OkgSUSJ&*3>IU8W+o=A22s z#b82ufa*tF+?(9@7di@xZNPx`pm}-tcWe;7zuHx!Ji>3T4VamC!?8qSG-Z0#j*~eB z^neOq1GwK@xmrA_P}ITiu!fNo%=KnW-Kc^p0F;*-%wN>HYD(fED0(6Kmb`~x8ugCTTHW*tE8F3~N|Z8z%wsZ&Vmb@PJ2P*f zm~9i)sf2lXJr`^AfqEcm*4n@rjG~*x7#X+1_U1Meh4=on*y9K~#vzM7q4)swl~~wVcQt3Whrx zCH*dGOf|}zkJ!S&hh(Uw+IlAD=y3A0Y)w^TDiJjxqM$ZjD=FOp&BqPi!D$c%R1(b# z#0pTVE-cu;cK&<#*u)Mqc{W8J4~S&_Vpaip9+T;vJ+u=oAhE8uPdu;c@@$vQmF-vAcR$-%+&AzG|C2yjr>y^pJ@Wa54TKCQvZ z*2y}q$LW!X=Bkw+SD%3y^)SPP-RboXgA0goK3JfLZYP z2tXd-GK63PAUHiejZVVv3YcvS49FtM;UOz1audVBk!2^Ptudd9DJUoae=eKfjiLBv zwpczRBLg@B&;exz`iT`4*sv7jIaGTAy@qB-%QCgBH(w_b794Lv4%c%E+Ci-zjj+YC^}r`+*<@vHGh7u# zL@db}`^MnntFAQsySS(=C)l&w9}cYWv=QDqM0wm5t$k2U2fzQnJv?}tT0D5+Cd~a( zRfCYWz)jj%mFocjAED&0;QjM+44u|W5@Jx@Zg0KPJt<#G zhMJB!+6DLh>59{fq5ZS`xxEU>suwsTK+L@9R>5MjROF@oa#bSLrr0PImZ8ZC2k6U1 z&ui)#P#Cs=D7MVuRg7W+tLW#Ox9akm{QZ6I)+N~A4XAj$U-Bn+b%N0&e2Bf8j=teG zwQ1Rrk)wtb#97= zYoW3B!xY$UAZMUl6r_NS2(6sycQ&g9_G5>82n~k|eSGE!=)a0u>&Gr@fDT?-5{{3F zCt9erG<#-~#jLQsllKBRK%+YupWDk?F^7B#eU$zxYHO5?cj3yA51$e~cGOo`wUZ@$M52XaKHadqq~_HV&_8Ktas^{lQ{)Kk{#Iyl(PniAgYK<_$N_ruh5|%%_5pRn>dX7f zyZiDsw`gSXGUT(Cl<7L9FHRVP6EkA<1pEoyhf(A30aIg;BDbnkolJr*@YyF)966VgbRDh@glYvgi z@wYb+rv2UuWVACF>4~BCN{3XtnmslSrKaySMSvrjps6Wxbus+|B%-YngH!0;dzw#6 z(Rr4hp8f!|YO%!uuNP?HhV!7go?9A!#x;0xgU%0`oQYy!L=-0(#_rrYIOaU9f%2K~ zVmr;>r&S+JN=jnCCdBp*EsrK%!w|d>1)A(m*=6qxY{LqP1pkg|9pqEcYMU`Oy%?gE)gb_=ET7Xy+ z8v+YLWE&>^$oUYDta1M%{Ffy0%t} + +Neo4j metadata will be ingested into DataHub using +`CALL apoc.meta.schema() YIELD value UNWIND keys(value) AS key RETURN key, value[key] AS value;` +The data that is returned will be parsed +and will be displayed as Nodes and Relationships in DataHub. Each object will be tagged with describing what kind of DataHub +object it is. The defaults are 'Node' and 'Relationship'. These tag values can be overwritten in the recipe. + + + +## Metadata Ingestion Quickstart + +### Prerequisites + +In order to ingest metadata from Neo4j, you will need: + +* Neo4j instance with APOC installed + diff --git a/metadata-ingestion/docs/sources/neo4j/neo4j_recipe.yml b/metadata-ingestion/docs/sources/neo4j/neo4j_recipe.yml new file mode 100644 index 00000000000000..463d65e7ba323b --- /dev/null +++ b/metadata-ingestion/docs/sources/neo4j/neo4j_recipe.yml @@ -0,0 +1,12 @@ +source: + type: 'neo4j' + config: + uri: 'neo4j+ssc://host:7687' + username: 'neo4j' + password: 'password' + env: 'PROD' + +sink: + type: "datahub-rest" + config: + server: 'http://localhost:8080' \ No newline at end of file diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index d7e056b31370df..c6d55fb5bcc56e 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -525,6 +525,7 @@ "qlik-sense": sqlglot_lib | {"requests", "websocket-client"}, "sigma": sqlglot_lib | {"requests"}, "sac": sac, + "neo4j": {"pandas", "neo4j"}, } # This is mainly used to exclude plugins from the Docker image. @@ -673,6 +674,7 @@ "sigma", "sac", "cassandra", + "neo4j", ] if plugin for dependency in plugins[plugin] @@ -792,6 +794,7 @@ "sigma = datahub.ingestion.source.sigma.sigma:SigmaSource", "sac = datahub.ingestion.source.sac.sac:SACSource", "cassandra = datahub.ingestion.source.cassandra.cassandra:CassandraSource", + "neo4j = datahub.ingestion.source.neo4j.neo4j_source:Neo4jSource", ], "datahub.ingestion.transformer.plugins": [ "pattern_cleanup_ownership = datahub.ingestion.transformer.pattern_cleanup_ownership:PatternCleanUpOwnership", diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 9fbb15500a863c..a5eecf198a9b49 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -22,6 +22,8 @@ class DatasetSubTypes(StrEnum): SAC_MODEL = "Model" SAC_IMPORT_DATA_MODEL = "Import Data Model" SAC_LIVE_DATA_MODEL = "Live Data Model" + NEO4J_NODE = "Neo4j Node" + NEO4J_RELATIONSHIP = "Neo4j Relationship" # TODO: Create separate entity... NOTEBOOK = "Notebook" diff --git a/metadata-ingestion/src/datahub/ingestion/source/neo4j/__init__.py b/metadata-ingestion/src/datahub/ingestion/source/neo4j/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py b/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py new file mode 100644 index 00000000000000..2c9107b967e4f8 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/neo4j/neo4j_source.py @@ -0,0 +1,331 @@ +import logging +import time +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional, Type, Union + +import pandas as pd +from neo4j import GraphDatabase +from pydantic.fields import Field + +from datahub.configuration.source_common import EnvConfigMixin +from datahub.emitter.mce_builder import make_data_platform_urn, make_dataset_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.api.decorators import ( + SupportStatus, + config_class, + platform_name, + support_status, +) +from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.common.subtypes import DatasetSubTypes +from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaFieldDataType +from datahub.metadata.schema_classes import ( + AuditStampClass, + BooleanTypeClass, + DatasetPropertiesClass, + DateTypeClass, + NullTypeClass, + NumberTypeClass, + OtherSchemaClass, + SchemaFieldClass, + SchemaMetadataClass, + StringTypeClass, + SubTypesClass, + UnionTypeClass, +) + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +_type_mapping: Dict[Union[Type, str], Type] = { + "list": UnionTypeClass, + "boolean": BooleanTypeClass, + "integer": NumberTypeClass, + "local_date_time": DateTypeClass, + "float": NumberTypeClass, + "string": StringTypeClass, + "date": DateTypeClass, + "node": StringTypeClass, + "relationship": StringTypeClass, +} + + +class Neo4jConfig(EnvConfigMixin): + username: str = Field(description="Neo4j Username") + password: str = Field(description="Neo4j Password") + uri: str = Field(description="The URI for the Neo4j server") + env: str = Field(description="Neo4j env") + + +@dataclass +class Neo4jSourceReport(SourceReport): + obj_failures: int = 0 + obj_created: int = 0 + + +@platform_name("Neo4j", id="neo4j") +@config_class(Neo4jConfig) +@support_status(SupportStatus.CERTIFIED) +class Neo4jSource(Source): + NODE = "node" + RELATIONSHIP = "relationship" + PLATFORM = "neo4j" + + def __init__(self, ctx: PipelineContext, config: Neo4jConfig): + self.ctx = ctx + self.config = config + self.report = Neo4jSourceReport() + + @classmethod + def create(cls, config_dict, ctx): + config = Neo4jConfig.parse_obj(config_dict) + return cls(ctx, config) + + def get_field_type(self, attribute_type: Union[type, str]) -> SchemaFieldDataType: + type_class: type = _type_mapping.get(attribute_type, NullTypeClass) + return SchemaFieldDataType(type=type_class()) + + def get_schema_field_class( + self, col_name: str, col_type: str, **kwargs: Any + ) -> SchemaFieldClass: + if kwargs["obj_type"] == self.NODE and col_type == self.RELATIONSHIP: + col_type = self.NODE + else: + col_type = col_type + return SchemaFieldClass( + fieldPath=col_name, + type=self.get_field_type(col_type), + nativeDataType=col_type, + description=col_type.upper() + if col_type in (self.NODE, self.RELATIONSHIP) + else col_type, + lastModified=AuditStampClass( + time=round(time.time() * 1000), actor="urn:li:corpuser:ingestion" + ), + ) + + def add_properties( + self, + dataset: str, + description: Optional[str] = None, + custom_properties: Optional[Dict[str, str]] = None, + ) -> MetadataChangeProposalWrapper: + dataset_properties = DatasetPropertiesClass( + description=description, + customProperties=custom_properties, + ) + return MetadataChangeProposalWrapper( + entityUrn=make_dataset_urn( + platform=self.PLATFORM, name=dataset, env=self.config.env + ), + aspect=dataset_properties, + ) + + def generate_neo4j_object( + self, dataset: str, columns: list, obj_type: Optional[str] = None + ) -> MetadataChangeProposalWrapper: + try: + fields = [ + self.get_schema_field_class(key, value.lower(), obj_type=obj_type) + for d in columns + for key, value in d.items() + ] + mcp = MetadataChangeProposalWrapper( + entityUrn=make_dataset_urn( + platform=self.PLATFORM, name=dataset, env=self.config.env + ), + aspect=SchemaMetadataClass( + schemaName=dataset, + platform=make_data_platform_urn(self.PLATFORM), + version=0, + hash="", + platformSchema=OtherSchemaClass(rawSchema=""), + lastModified=AuditStampClass( + time=round(time.time() * 1000), + actor="urn:li:corpuser:ingestion", + ), + fields=fields, + ), + ) + self.report.obj_created += 1 + except Exception as e: + log.error(e) + self.report.obj_failures += 1 + return mcp + + def get_neo4j_metadata(self, query: str) -> pd.DataFrame: + driver = GraphDatabase.driver( + self.config.uri, auth=(self.config.username, self.config.password) + ) + """ + This process retrieves the metadata for Neo4j objects using an APOC query, which returns a dictionary + with two columns: key and value. The key represents the Neo4j object, while the value contains the + corresponding metadata. + + When data is returned from Neo4j, much of the relationship metadata is stored with the relevant node's + metadata. Consequently, the objects are organized into two separate dataframes: one for nodes and one for + relationships. + + In the node dataframe, several fields are extracted and added as new columns. Similarly, in the relationship + dataframe, certain fields are parsed out, while others require metadata from the nodes dataframe. + + Once the data is parsed and these two dataframes are created, we combine a subset of their columns into a + single dataframe, which will be used to create the DataHub objects. + + See the docs for examples of metadata: metadata-ingestion/docs/sources/neo4j/neo4j.md + """ + try: + log.info(f"{query}") + with driver.session() as session: + result = session.run(query) + data = [record for record in result] + log.info("Closing Neo4j driver") + driver.close() + + node_df = self.process_nodes(data) + rel_df = self.process_relationships(data, node_df) + + union_cols = ["key", "obj_type", "property_data_types", "description"] + df = pd.concat([node_df[union_cols], rel_df[union_cols]]) + except Exception as e: + self.report.failure( + message="Failed to get neo4j metadata", + exc=e, + ) + + return df + + def process_nodes(self, data: list) -> pd.DataFrame: + nodes = [record for record in data if record["value"]["type"] == self.NODE] + node_df = pd.DataFrame( + nodes, + columns=["key", "value"], + ) + node_df["obj_type"] = node_df["value"].apply( + lambda record: self.get_obj_type(record) + ) + node_df["relationships"] = node_df["value"].apply( + lambda record: self.get_relationships(record) + ) + node_df["properties"] = node_df["value"].apply( + lambda record: self.get_properties(record) + ) + node_df["property_data_types"] = node_df["properties"].apply( + lambda record: self.get_property_data_types(record) + ) + node_df["description"] = node_df.apply( + lambda record: self.get_node_description(record, node_df), axis=1 + ) + return node_df + + def process_relationships(self, data: list, node_df: pd.DataFrame) -> pd.DataFrame: + rels = [ + record for record in data if record["value"]["type"] == self.RELATIONSHIP + ] + rel_df = pd.DataFrame(rels, columns=["key", "value"]) + rel_df["obj_type"] = rel_df["value"].apply( + lambda record: self.get_obj_type(record) + ) + rel_df["properties"] = rel_df["value"].apply( + lambda record: self.get_properties(record) + ) + rel_df["property_data_types"] = rel_df["properties"].apply( + lambda record: self.get_property_data_types(record) + ) + rel_df["description"] = rel_df.apply( + lambda record: self.get_rel_descriptions(record, node_df), axis=1 + ) + return rel_df + + def get_obj_type(self, record: dict) -> str: + return record["type"] + + def get_rel_descriptions(self, record: dict, df: pd.DataFrame) -> str: + descriptions = [] + for _, row in df.iterrows(): + relationships = row.get("relationships", {}) + for relationship, props in relationships.items(): + if record["key"] == relationship: + if props["direction"] == "in": + for prop in props["labels"]: + descriptions.append( + f"({row['key']})-[{record['key']}]->({prop})" + ) + return "\n".join(descriptions) + + def get_node_description(self, record: dict, df: pd.DataFrame) -> str: + descriptions = [] + for _, row in df.iterrows(): + if record["key"] == row["key"]: + for relationship, props in row["relationships"].items(): + direction = props["direction"] + for node in set(props["labels"]): + if direction == "in": + descriptions.append( + f"({row['key']})<-[{relationship}]-({node})" + ) + elif direction == "out": + descriptions.append( + f"({row['key']})-[{relationship}]->({node})" + ) + + return "\n".join(descriptions) + + def get_property_data_types(self, record: dict) -> List[dict]: + return [{k: v["type"]} for k, v in record.items()] + + def get_properties(self, record: dict) -> str: + return record["properties"] + + def get_relationships(self, record: dict) -> dict: + return record.get("relationships", None) + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + df = self.get_neo4j_metadata( + "CALL apoc.meta.schema() YIELD value UNWIND keys(value) AS key RETURN key, value[key] AS value;" + ) + for index, row in df.iterrows(): + try: + yield MetadataWorkUnit( + id=row["key"], + mcp=self.generate_neo4j_object( + columns=row["property_data_types"], + dataset=row["key"], + ), + is_primary_source=True, + ) + + yield MetadataWorkUnit( + id=row["key"], + mcp=MetadataChangeProposalWrapper( + entityUrn=make_dataset_urn( + platform=self.PLATFORM, + name=row["key"], + env=self.config.env, + ), + aspect=SubTypesClass( + typeNames=[ + DatasetSubTypes.NEO4J_NODE + if row["obj_type"] == self.NODE + else DatasetSubTypes.NEO4J_RELATIONSHIP + ] + ), + ), + ) + + yield MetadataWorkUnit( + id=row["key"], + mcp=self.add_properties( + dataset=row["key"], + custom_properties=None, + description=row["description"], + ), + ) + + except Exception as e: + raise e + + def get_report(self): + return self.report diff --git a/metadata-ingestion/tests/unit/test_neo4j_source.py b/metadata-ingestion/tests/unit/test_neo4j_source.py new file mode 100644 index 00000000000000..62586718e86067 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_neo4j_source.py @@ -0,0 +1,221 @@ +import unittest +from pathlib import Path + +import pandas as pd +import pytest + +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.neo4j.neo4j_source import Neo4jConfig, Neo4jSource + + +@pytest.fixture +def tracking_uri(tmp_path: Path) -> str: + # return str(tmp_path / "neo4j") + return "neo4j+ssc://host:7687" + + +@pytest.fixture +def source(tracking_uri: str) -> Neo4jSource: + return Neo4jSource( + ctx=PipelineContext(run_id="neo4j-test"), + config=Neo4jConfig( + uri=tracking_uri, env="Prod", username="test", password="test" + ), + ) + + +def data(): + return [ + { + "key": "Node_1", + "value": { + "count": 433026, + "relationships": { + "RELATIONSHIP_1": { + "count": 1, + "properties": { + "Relationship1_Property1": { + "existence": False, + "type": "STRING", + "indexed": False, + "array": False, + } + }, + "direction": "in", + "labels": ["Node_2"], + } + }, + "RELATIONSHIP_2": { + "count": 2, + "properties": { + "Relationship2_Property1": { + "existence": False, + "type": "STRING", + "indexed": False, + "array": False, + } + }, + "direction": "in", + "labels": ["Node_3"], + }, + "type": "node", + "properties": { + "Node1_Property1": { + "existence": False, + "type": "DATE", + "indexed": False, + "unique": False, + }, + "Node1_Property2": { + "existence": False, + "type": "STRING", + "indexed": False, + "unique": False, + }, + "Node1_Property3": { + "existence": False, + "type": "STRING", + "indexed": False, + "unique": False, + }, + }, + "labels": [], + }, + }, + { + "key": "Node_2", + "value": { + "count": 3, + "relationships": { + "RELATIONSHIP_1": { + "count": 1, + "properties": { + "Relationship1_Property1": { + "existence": False, + "type": "STRING", + "indexed": False, + "array": False, + } + }, + "direction": "out", + "labels": ["Node_2"], + } + }, + "type": "node", + "properties": { + "Node2_Property1": { + "existence": False, + "type": "DATE", + "indexed": False, + "unique": False, + }, + "Node2_Property2": { + "existence": False, + "type": "STRING", + "indexed": False, + "unique": False, + }, + "Node2_Property3": { + "existence": False, + "type": "STRING", + "indexed": False, + "unique": False, + }, + }, + "labels": [], + }, + }, + { + "key": "RELATIONSHIP_1", + "value": { + "count": 4, + "type": "relationship", + "properties": { + "Relationship1_Property1": { + "existence": False, + "type": "STRING", + "indexed": False, + "array": False, + } + }, + }, + }, + ] + + +def test_process_nodes(source): + df = source.process_nodes(data=data()) + assert type(df) is pd.DataFrame + + +def test_process_relationships(source): + df = source.process_relationships( + data=data(), node_df=source.process_nodes(data=data()) + ) + assert type(df) is pd.DataFrame + + +def test_get_obj_type(source): + results = data() + assert source.get_obj_type(results[0]["value"]) == "node" + assert source.get_obj_type(results[1]["value"]) == "node" + assert source.get_obj_type(results[2]["value"]) == "relationship" + + +def test_get_node_description(source): + results = data() + df = source.process_nodes(data=data()) + assert ( + source.get_node_description(results[0], df) + == "(Node_1)<-[RELATIONSHIP_1]-(Node_2)" + ) + assert ( + source.get_node_description(results[1], df) + == "(Node_2)-[RELATIONSHIP_1]->(Node_2)" + ) + + +def test_get_property_data_types(source): + results = data() + assert source.get_property_data_types(results[0]["value"]["properties"]) == [ + {"Node1_Property1": "DATE"}, + {"Node1_Property2": "STRING"}, + {"Node1_Property3": "STRING"}, + ] + assert source.get_property_data_types(results[1]["value"]["properties"]) == [ + {"Node2_Property1": "DATE"}, + {"Node2_Property2": "STRING"}, + {"Node2_Property3": "STRING"}, + ] + assert source.get_property_data_types(results[2]["value"]["properties"]) == [ + {"Relationship1_Property1": "STRING"} + ] + + +def test_get_properties(source): + results = data() + assert list(source.get_properties(results[0]["value"]).keys()) == [ + "Node1_Property1", + "Node1_Property2", + "Node1_Property3", + ] + assert list(source.get_properties(results[1]["value"]).keys()) == [ + "Node2_Property1", + "Node2_Property2", + "Node2_Property3", + ] + assert list(source.get_properties(results[2]["value"]).keys()) == [ + "Relationship1_Property1" + ] + + +def test_get_relationships(source): + results = data() + record = list( + results[0]["value"]["relationships"].keys() + ) # Get the first key from the dict_keys + assert record == ["RELATIONSHIP_1"] + + +if __name__ == "__main__": + unittest.main() diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml index 1625df4a99540d..0b3d815c710980 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml @@ -727,3 +727,14 @@ displayName: Cassandra type: KEY_VALUE_STORE logoUrl: "/assets/platforms/cassandralogo.png" +- entityUrn: urn:li:dataPlatform:neo4j + entityType: dataPlatform + aspectName: dataPlatformInfo + changeType: UPSERT + aspect: + datasetNameDelimiter: "." + name: neo4j + displayName: Neo4j + type: OTHERS + logoUrl: "/assets/platforms/neo4j.png" + From b538d4bc7397d68462ad721fe56718b30d9885b8 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Mon, 2 Dec 2024 15:02:47 +0530 Subject: [PATCH 106/174] fix(ingestion/dremio): Fixed lineage view for dremio EE (#11990) --- .../ingestion/source/dremio/dremio_source.py | 6 +- .../dremio/dremio_mces_golden.json | 2092 +++++++++++------ 2 files changed, 1423 insertions(+), 675 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py index f814108c377605..319290d25169af 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py @@ -396,10 +396,12 @@ def process_dataset( ): yield dremio_mcp # Check if the emitted aspect is SchemaMetadataClass - if isinstance(dremio_mcp.metadata, SchemaMetadataClass): + if isinstance( + dremio_mcp.metadata, MetadataChangeProposalWrapper + ) and isinstance(dremio_mcp.metadata.aspect, SchemaMetadataClass): self.sql_parsing_aggregator.register_schema( urn=dataset_urn, - schema=dremio_mcp.metadata, + schema=dremio_mcp.metadata.aspect, ) if dataset_info.dataset_type == DremioDatasetType.VIEW: diff --git a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json index df89255c300481..3a8fce62f4bb36 100644 --- a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json +++ b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json @@ -15,7 +15,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -31,7 +31,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -65,7 +65,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -85,7 +85,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -105,7 +105,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -121,7 +121,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -139,7 +139,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -155,7 +155,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -175,7 +175,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -195,7 +195,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -211,7 +211,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -229,7 +229,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -245,7 +245,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -265,7 +265,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -285,7 +285,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -301,7 +301,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -319,7 +319,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -335,7 +335,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -355,7 +355,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -375,7 +375,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -391,7 +391,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -409,7 +409,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -425,7 +425,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -445,7 +445,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -465,7 +465,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -481,7 +481,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -497,7 +497,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -515,7 +515,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -531,7 +531,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -555,7 +555,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -575,7 +575,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -591,7 +591,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -607,7 +607,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -625,7 +625,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -641,7 +641,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -665,7 +665,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -685,7 +685,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -701,7 +701,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -717,7 +717,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -735,7 +735,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -751,7 +751,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -775,7 +775,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -795,7 +795,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -811,7 +811,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -827,7 +827,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -845,7 +845,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -861,7 +861,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -885,7 +885,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -905,7 +905,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -921,7 +921,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -937,7 +937,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -955,7 +955,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -971,7 +971,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -995,7 +995,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1015,7 +1015,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1031,7 +1031,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1047,7 +1047,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1065,7 +1065,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1081,7 +1081,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1105,7 +1105,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1125,7 +1125,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1141,7 +1141,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1157,7 +1157,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1175,7 +1175,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1191,7 +1191,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1215,7 +1215,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1235,7 +1235,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1251,7 +1251,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1267,7 +1267,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1285,7 +1285,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1301,7 +1301,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1325,7 +1325,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1345,7 +1345,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1361,7 +1361,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1377,7 +1377,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1395,7 +1395,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1411,7 +1411,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1435,7 +1435,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1455,7 +1455,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1471,7 +1471,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1487,7 +1487,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1505,7 +1505,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1521,7 +1521,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1549,7 +1549,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1569,7 +1569,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1585,7 +1585,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1601,7 +1601,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1619,7 +1619,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1635,7 +1635,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1663,7 +1663,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1683,7 +1683,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1699,7 +1699,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1715,7 +1715,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1733,7 +1733,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1749,7 +1749,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1781,7 +1781,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1801,7 +1801,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1817,7 +1817,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1833,7 +1833,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1851,7 +1851,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1867,7 +1867,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1903,7 +1903,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1927,7 +1927,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1945,7 +1945,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1961,7 +1961,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1977,7 +1977,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1995,7 +1995,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2025,7 +2025,7 @@ }, "fields": [ { - "fieldPath": "A", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -2037,7 +2037,7 @@ "isPartOfKey": false }, { - "fieldPath": "I", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -2061,7 +2061,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -2073,7 +2073,7 @@ "isPartOfKey": false }, { - "fieldPath": "F", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -2085,7 +2085,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -2097,7 +2097,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -2109,7 +2109,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -2121,7 +2121,7 @@ "isPartOfKey": false }, { - "fieldPath": "B", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -2137,7 +2137,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2153,7 +2153,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2177,7 +2177,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2201,7 +2201,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2219,7 +2219,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2235,7 +2235,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2251,7 +2251,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2269,7 +2269,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2299,31 +2299,31 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "email_address", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "company", + "fieldPath": "priority", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "float(24)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_name", + "fieldPath": "company", "nullable": true, "type": { "type": { @@ -2335,31 +2335,31 @@ "isPartOfKey": false }, { - "fieldPath": "first_name", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "priority", + "fieldPath": "first_name", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "float(24)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "email_address", + "fieldPath": "last_name", "nullable": true, "type": { "type": { @@ -2375,7 +2375,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2391,7 +2391,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2419,7 +2419,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2443,7 +2443,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2461,7 +2461,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2477,7 +2477,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2493,7 +2493,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2511,7 +2511,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2541,7 +2541,7 @@ }, "fields": [ { - "fieldPath": "urn", + "fieldPath": "createdby", "nullable": true, "type": { "type": { @@ -2553,7 +2553,7 @@ "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "createdfor", "nullable": true, "type": { "type": { @@ -2565,19 +2565,19 @@ "isPartOfKey": false }, { - "fieldPath": "version", + "fieldPath": "urn", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "metadata", + "fieldPath": "aspect", "nullable": true, "type": { "type": { @@ -2589,19 +2589,19 @@ "isPartOfKey": false }, { - "fieldPath": "createdon", + "fieldPath": "version", "nullable": true, "type": { "type": { - "com.linkedin.schema.DateType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "timestamp(23)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdby", + "fieldPath": "metadata", "nullable": true, "type": { "type": { @@ -2613,14 +2613,14 @@ "isPartOfKey": false }, { - "fieldPath": "createdfor", + "fieldPath": "createdon", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.DateType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "timestamp(23)", "recursive": false, "isPartOfKey": false } @@ -2629,7 +2629,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2645,7 +2645,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2673,7 +2673,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2697,7 +2697,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2715,7 +2715,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2731,7 +2731,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2747,7 +2747,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2765,7 +2765,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2795,19 +2795,19 @@ }, "fields": [ { - "fieldPath": "path", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "urn", "nullable": true, "type": { "type": { @@ -2819,26 +2819,26 @@ "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "longVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "stringVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, @@ -2855,7 +2855,7 @@ "isPartOfKey": false }, { - "fieldPath": "stringVal", + "fieldPath": "path", "nullable": true, "type": { "type": { @@ -2867,14 +2867,14 @@ "isPartOfKey": false }, { - "fieldPath": "longVal", + "fieldPath": "aspect", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false } @@ -2883,7 +2883,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2899,7 +2899,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2927,7 +2927,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2951,7 +2951,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2969,7 +2969,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2985,7 +2985,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3001,7 +3001,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3019,7 +3019,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3049,19 +3049,19 @@ }, "fields": [ { - "fieldPath": "doubleVal", + "fieldPath": "id", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "double(53)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "path", + "fieldPath": "urn", "nullable": true, "type": { "type": { @@ -3073,7 +3073,7 @@ "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "path", "nullable": true, "type": { "type": { @@ -3085,14 +3085,14 @@ "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "doubleVal", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "double(53)", "recursive": false, "isPartOfKey": false } @@ -3101,7 +3101,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3117,7 +3117,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3145,7 +3145,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3169,7 +3169,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3187,7 +3187,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3203,7 +3203,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3219,7 +3219,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3237,7 +3237,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3267,7 +3267,7 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "customer_id", "nullable": true, "type": { "type": { @@ -3291,7 +3291,7 @@ "isPartOfKey": false }, { - "fieldPath": "customer_id", + "fieldPath": "id", "nullable": true, "type": { "type": { @@ -3307,7 +3307,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3323,7 +3323,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3351,7 +3351,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3375,7 +3375,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3393,7 +3393,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3409,7 +3409,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3425,7 +3425,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3443,7 +3443,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3473,7 +3473,7 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "salary", "nullable": true, "type": { "type": { @@ -3485,31 +3485,31 @@ "isPartOfKey": false }, { - "fieldPath": "name", + "fieldPath": "age", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "age", + "fieldPath": "name", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "salary", + "fieldPath": "id", "nullable": true, "type": { "type": { @@ -3525,7 +3525,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3541,7 +3541,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3569,7 +3569,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3593,7 +3593,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3611,7 +3611,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3627,7 +3627,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3643,7 +3643,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3725,7 +3725,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3741,7 +3741,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3766,7 +3766,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3794,7 +3794,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3818,7 +3818,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3836,7 +3836,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3852,7 +3852,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3868,7 +3868,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3910,7 +3910,7 @@ "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "createdfor", "nullable": true, "type": { "type": { @@ -3922,55 +3922,55 @@ "isPartOfKey": false }, { - "fieldPath": "version", + "fieldPath": "createdby", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "metadata", + "fieldPath": "createdon", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.DateType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "timestamp(23)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdon", + "fieldPath": "metadata", "nullable": true, "type": { "type": { - "com.linkedin.schema.DateType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "timestamp(23)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdby", + "fieldPath": "version", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdfor", + "fieldPath": "aspect", "nullable": true, "type": { "type": { @@ -3986,7 +3986,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4002,7 +4002,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4027,7 +4027,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4055,7 +4055,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4079,7 +4079,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4097,7 +4097,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4113,7 +4113,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4129,7 +4129,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4247,7 +4247,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4263,7 +4263,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4288,7 +4288,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4316,7 +4316,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4340,7 +4340,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4358,7 +4358,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4374,7 +4374,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4390,7 +4390,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4432,38 +4432,38 @@ "isPartOfKey": false }, { - "fieldPath": "doubleVal", + "fieldPath": "id", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "double(53)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "urn", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "doubleVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "double(53)", "recursive": false, "isPartOfKey": false } @@ -4472,7 +4472,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4488,7 +4488,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4513,7 +4513,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4541,7 +4541,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4565,7 +4565,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4583,7 +4583,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4599,7 +4599,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4615,7 +4615,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4721,7 +4721,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4737,7 +4737,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4762,7 +4762,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4790,7 +4790,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4814,7 +4814,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4832,7 +4832,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4848,7 +4848,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4864,7 +4864,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4894,7 +4894,7 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "customer_id", "nullable": true, "type": { "type": { @@ -4906,26 +4906,26 @@ "isPartOfKey": false }, { - "fieldPath": "description", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "customer_id", + "fieldPath": "description", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false } @@ -4934,7 +4934,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4950,7 +4950,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4975,7 +4975,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5003,7 +5003,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5027,7 +5027,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5045,7 +5045,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5061,7 +5061,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5077,7 +5077,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5107,7 +5107,7 @@ }, "fields": [ { - "fieldPath": "B", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -5119,7 +5119,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -5131,7 +5131,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "H", "nullable": true, "type": { "type": { @@ -5143,7 +5143,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -5155,7 +5155,7 @@ "isPartOfKey": false }, { - "fieldPath": "F", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -5167,7 +5167,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -5179,7 +5179,7 @@ "isPartOfKey": false }, { - "fieldPath": "H", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -5191,7 +5191,7 @@ "isPartOfKey": false }, { - "fieldPath": "I", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -5203,7 +5203,7 @@ "isPartOfKey": false }, { - "fieldPath": "A", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -5219,7 +5219,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5235,7 +5235,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5260,7 +5260,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5288,7 +5288,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5312,7 +5312,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5330,7 +5330,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5346,7 +5346,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5362,7 +5362,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5392,7 +5392,7 @@ }, "fields": [ { - "fieldPath": "I", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -5404,7 +5404,7 @@ "isPartOfKey": false }, { - "fieldPath": "A", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -5416,7 +5416,7 @@ "isPartOfKey": false }, { - "fieldPath": "B", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -5428,7 +5428,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -5440,7 +5440,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -5452,7 +5452,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -5464,7 +5464,7 @@ "isPartOfKey": false }, { - "fieldPath": "F", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -5476,7 +5476,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "H", "nullable": true, "type": { "type": { @@ -5488,7 +5488,7 @@ "isPartOfKey": false }, { - "fieldPath": "H", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -5552,7 +5552,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5568,7 +5568,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5593,7 +5593,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5625,7 +5625,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5649,7 +5649,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5667,7 +5667,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5683,7 +5683,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5699,7 +5699,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5781,7 +5781,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5797,7 +5797,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5822,7 +5822,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5854,7 +5854,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5878,7 +5878,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5896,7 +5896,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5912,7 +5912,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5928,7 +5928,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6070,7 +6070,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6086,7 +6086,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6111,7 +6111,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6151,7 +6151,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6175,12 +6175,91 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdfor)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdby)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdon)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdon)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),metadata)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),version)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),version)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),aspect)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),aspect)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6204,12 +6283,91 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),doubleVal)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),aspect)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),aspect)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),path)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),path)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),longVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),longVal)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),stringVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6233,12 +6391,58 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),path)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),doubleVal)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6262,12 +6466,80 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),company)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),company)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),email_address)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),email_address)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),priority)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),priority)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6291,12 +6563,47 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),description)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),description)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6320,12 +6627,58 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),age)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),salary)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),salary)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6349,12 +6702,157 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6378,12 +6876,58 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6407,12 +6951,113 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6436,18 +7081,119 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6457,41 +7203,52 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 5, - "columnCount": 6, + "rowCount": 3834, + "columnCount": 9, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 5, - "nullCount": 0, - "mean": "3.0", - "stdev": "1.5811388300841898" + "fieldPath": "F", + "uniqueCount": 61, + "nullCount": 0 }, { - "fieldPath": "company", - "uniqueCount": 5, + "fieldPath": "G", + "uniqueCount": 40, "nullCount": 0 }, { - "fieldPath": "last_name", - "uniqueCount": 5, + "fieldPath": "H", + "uniqueCount": 91, "nullCount": 0 }, { - "fieldPath": "first_name", - "uniqueCount": 5, + "fieldPath": "I", + "uniqueCount": 85, "nullCount": 0 }, { - "fieldPath": "priority", - "uniqueCount": 3, - "nullCount": 1, - "mean": "4.175000011920929", - "stdev": "0.4924429489953036" + "fieldPath": "A", + "uniqueCount": 2, + "nullCount": 0 }, { - "fieldPath": "email_address", - "uniqueCount": 5, + "fieldPath": "B", + "uniqueCount": 2, + "nullCount": 0 + }, + { + "fieldPath": "C", + "uniqueCount": 3834, + "nullCount": 0 + }, + { + "fieldPath": "D", + "uniqueCount": 76, + "nullCount": 0 + }, + { + "fieldPath": "E", + "uniqueCount": 192, "nullCount": 0 } ] @@ -6499,13 +7256,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6516,7 +7273,7 @@ "type": "FULL_TABLE" }, "rowCount": 0, - "columnCount": 7, + "columnCount": 4, "fieldProfiles": [ { "fieldPath": "path", @@ -6524,7 +7281,7 @@ "nullCount": 0 }, { - "fieldPath": "aspect", + "fieldPath": "id", "uniqueCount": 0, "nullCount": 0 }, @@ -6533,38 +7290,23 @@ "uniqueCount": 0, "nullCount": 0 }, - { - "fieldPath": "id", - "uniqueCount": 0, - "nullCount": 0 - }, { "fieldPath": "doubleVal", "uniqueCount": 0, "nullCount": 0 - }, - { - "fieldPath": "stringVal", - "uniqueCount": 0, - "nullCount": 0 - }, - { - "fieldPath": "longVal", - "uniqueCount": 0, - "nullCount": 0 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6575,20 +7317,40 @@ "type": "FULL_TABLE" }, "rowCount": 0, - "columnCount": 3, + "columnCount": 7, "fieldProfiles": [ + { + "fieldPath": "doubleVal", + "uniqueCount": 0, + "nullCount": 0 + }, { "fieldPath": "id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "description", + "fieldPath": "urn", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "customer_id", + "fieldPath": "aspect", + "uniqueCount": 0, + "nullCount": 0 + }, + { + "fieldPath": "path", + "uniqueCount": 0, + "nullCount": 0 + }, + { + "fieldPath": "longVal", + "uniqueCount": 0, + "nullCount": 0 + }, + { + "fieldPath": "stringVal", "uniqueCount": 0, "nullCount": 0 } @@ -6597,13 +7359,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6617,17 +7379,17 @@ "columnCount": 3, "fieldProfiles": [ { - "fieldPath": "id", + "fieldPath": "customer_id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "description", + "fieldPath": "id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "customer_id", + "fieldPath": "description", "uniqueCount": 0, "nullCount": 0 } @@ -6636,13 +7398,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6656,16 +7418,11 @@ "columnCount": 4, "fieldProfiles": [ { - "fieldPath": "id", + "fieldPath": "salary", "uniqueCount": 4, "nullCount": 0, - "mean": "2.5", - "stdev": "1.2909944487358056" - }, - { - "fieldPath": "name", - "uniqueCount": 4, - "nullCount": 0 + "mean": "65000.0", + "stdev": "12909.944487358056" }, { "fieldPath": "age", @@ -6675,24 +7432,29 @@ "stdev": "6.454972243679028" }, { - "fieldPath": "salary", + "fieldPath": "name", + "uniqueCount": 4, + "nullCount": 0 + }, + { + "fieldPath": "id", "uniqueCount": 4, "nullCount": 0, - "mean": "65000.0", - "stdev": "12909.944487358056" + "mean": "2.5", + "stdev": "1.2909944487358056" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6702,52 +7464,27 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 3834, - "columnCount": 9, + "rowCount": 0, + "columnCount": 4, "fieldProfiles": [ { - "fieldPath": "A", - "uniqueCount": 2, - "nullCount": 0 - }, - { - "fieldPath": "I", - "uniqueCount": 85, - "nullCount": 0 - }, - { - "fieldPath": "H", - "uniqueCount": 91, - "nullCount": 0 - }, - { - "fieldPath": "G", - "uniqueCount": 40, - "nullCount": 0 - }, - { - "fieldPath": "F", - "uniqueCount": 61, - "nullCount": 0 - }, - { - "fieldPath": "E", - "uniqueCount": 192, + "fieldPath": "id", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "D", - "uniqueCount": 76, + "fieldPath": "urn", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "C", - "uniqueCount": 3834, + "fieldPath": "path", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "B", - "uniqueCount": 2, + "fieldPath": "doubleVal", + "uniqueCount": 0, "nullCount": 0 } ] @@ -6755,13 +7492,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6771,27 +7508,41 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, - "columnCount": 4, + "rowCount": 5, + "columnCount": 6, "fieldProfiles": [ { - "fieldPath": "path", - "uniqueCount": 0, + "fieldPath": "email_address", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "doubleVal", - "uniqueCount": 0, + "fieldPath": "priority", + "uniqueCount": 3, + "nullCount": 1, + "mean": "4.175000011920929", + "stdev": "0.4924429489953036" + }, + { + "fieldPath": "company", + "uniqueCount": 5, "nullCount": 0 }, { "fieldPath": "id", - "uniqueCount": 0, + "uniqueCount": 5, + "nullCount": 0, + "mean": "3.0", + "stdev": "1.5811388300841898" + }, + { + "fieldPath": "first_name", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "urn", - "uniqueCount": 0, + "fieldPath": "last_name", + "uniqueCount": 5, "nullCount": 0 } ] @@ -6799,13 +7550,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6815,55 +7566,66 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 5, - "columnCount": 6, + "rowCount": 3834, + "columnCount": 9, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 5, - "nullCount": 0, - "mean": "3.0", - "stdev": "1.5811388300841898" + "fieldPath": "F", + "uniqueCount": 61, + "nullCount": 0 }, { - "fieldPath": "company", - "uniqueCount": 5, + "fieldPath": "G", + "uniqueCount": 40, "nullCount": 0 }, { - "fieldPath": "last_name", - "uniqueCount": 5, + "fieldPath": "H", + "uniqueCount": 91, "nullCount": 0 }, { - "fieldPath": "first_name", - "uniqueCount": 5, + "fieldPath": "I", + "uniqueCount": 85, "nullCount": 0 }, { - "fieldPath": "email_address", - "uniqueCount": 5, + "fieldPath": "A", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "priority", - "uniqueCount": 3, - "nullCount": 1, - "mean": "4.175000011920929", - "stdev": "0.4924429489953036" + "fieldPath": "B", + "uniqueCount": 2, + "nullCount": 0 + }, + { + "fieldPath": "C", + "uniqueCount": 3834, + "nullCount": 0 + }, + { + "fieldPath": "D", + "uniqueCount": 76, + "nullCount": 0 + }, + { + "fieldPath": "E", + "uniqueCount": 192, + "nullCount": 0 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6873,42 +7635,42 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, + "rowCount": 2, "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "doubleVal", - "uniqueCount": 0, + "fieldPath": "createdby", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "id", + "fieldPath": "createdfor", "uniqueCount": 0, - "nullCount": 0 + "nullCount": 2 }, { "fieldPath": "urn", - "uniqueCount": 0, + "uniqueCount": 1, "nullCount": 0 }, { "fieldPath": "aspect", - "uniqueCount": 0, + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "path", - "uniqueCount": 0, + "fieldPath": "version", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "longVal", - "uniqueCount": 0, + "fieldPath": "metadata", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "stringVal", - "uniqueCount": 0, + "fieldPath": "createdon", + "uniqueCount": 1, "nullCount": 0 } ] @@ -6916,13 +7678,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6932,47 +7694,56 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 4, - "columnCount": 4, + "rowCount": 2, + "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 4, - "nullCount": 0, - "mean": "2.5", - "stdev": "1.2909944487358056" + "fieldPath": "urn", + "uniqueCount": 1, + "nullCount": 0 }, { - "fieldPath": "name", - "uniqueCount": 4, + "fieldPath": "createdfor", + "uniqueCount": 0, + "nullCount": 2 + }, + { + "fieldPath": "createdby", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "age", - "uniqueCount": 4, - "nullCount": 0, - "mean": "32.5", - "stdev": "6.454972243679028" + "fieldPath": "createdon", + "uniqueCount": 1, + "nullCount": 0 }, { - "fieldPath": "salary", - "uniqueCount": 4, - "nullCount": 0, - "mean": "65000.0", - "stdev": "12909.944487358056" + "fieldPath": "metadata", + "uniqueCount": 2, + "nullCount": 0 + }, + { + "fieldPath": "version", + "uniqueCount": 1, + "nullCount": 0 + }, + { + "fieldPath": "aspect", + "uniqueCount": 2, + "nullCount": 0 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6982,86 +7753,47 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 10842, - "columnCount": 13, + "rowCount": 4, + "columnCount": 4, "fieldProfiles": [ { - "fieldPath": "I", - "uniqueCount": 8, - "nullCount": 0 - }, - { - "fieldPath": "A", - "uniqueCount": 9661, - "nullCount": 0 - }, - { - "fieldPath": "B", - "uniqueCount": 35, - "nullCount": 0 - }, - { - "fieldPath": "C", - "uniqueCount": 42, - "nullCount": 0 - }, - { - "fieldPath": "D", - "uniqueCount": 6003, - "nullCount": 0 - }, - { - "fieldPath": "E", - "uniqueCount": 463, - "nullCount": 0 - }, - { - "fieldPath": "F", - "uniqueCount": 23, - "nullCount": 0 - }, - { - "fieldPath": "G", - "uniqueCount": 5, - "nullCount": 0 - }, - { - "fieldPath": "H", - "uniqueCount": 94, - "nullCount": 0 - }, - { - "fieldPath": "J", - "uniqueCount": 121, - "nullCount": 0 + "fieldPath": "id", + "uniqueCount": 4, + "nullCount": 0, + "mean": "2.5", + "stdev": "1.2909944487358056" }, { - "fieldPath": "K", - "uniqueCount": 1379, + "fieldPath": "name", + "uniqueCount": 4, "nullCount": 0 }, { - "fieldPath": "L", - "uniqueCount": 2835, - "nullCount": 0 + "fieldPath": "age", + "uniqueCount": 4, + "nullCount": 0, + "mean": "32.5", + "stdev": "6.454972243679028" }, { - "fieldPath": "M", - "uniqueCount": 35, - "nullCount": 1 + "fieldPath": "salary", + "uniqueCount": 4, + "nullCount": 0, + "mean": "65000.0", + "stdev": "12909.944487358056" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7072,20 +7804,15 @@ "type": "FULL_TABLE" }, "rowCount": 0, - "columnCount": 4, + "columnCount": 3, "fieldProfiles": [ { - "fieldPath": "doubleVal", - "uniqueCount": 0, - "nullCount": 0 - }, - { - "fieldPath": "path", + "fieldPath": "customer_id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "urn", + "fieldPath": "description", "uniqueCount": 0, "nullCount": 0 }, @@ -7099,13 +7826,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7115,52 +7842,42 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 3834, - "columnCount": 9, + "rowCount": 0, + "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "B", - "uniqueCount": 2, - "nullCount": 0 - }, - { - "fieldPath": "C", - "uniqueCount": 3834, - "nullCount": 0 - }, - { - "fieldPath": "D", - "uniqueCount": 76, + "fieldPath": "id", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "E", - "uniqueCount": 192, + "fieldPath": "urn", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "F", - "uniqueCount": 61, + "fieldPath": "longVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "G", - "uniqueCount": 40, + "fieldPath": "stringVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "H", - "uniqueCount": 91, + "fieldPath": "doubleVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "I", - "uniqueCount": 85, + "fieldPath": "path", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "A", - "uniqueCount": 2, + "fieldPath": "aspect", + "uniqueCount": 0, "nullCount": 0 } ] @@ -7168,13 +7885,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7184,56 +7901,55 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 2, - "columnCount": 7, + "rowCount": 5, + "columnCount": 6, "fieldProfiles": [ { - "fieldPath": "urn", - "uniqueCount": 1, - "nullCount": 0 - }, - { - "fieldPath": "aspect", - "uniqueCount": 2, - "nullCount": 0 + "fieldPath": "id", + "uniqueCount": 5, + "nullCount": 0, + "mean": "3.0", + "stdev": "1.5811388300841898" }, { - "fieldPath": "version", - "uniqueCount": 1, + "fieldPath": "company", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "metadata", - "uniqueCount": 2, + "fieldPath": "last_name", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "createdon", - "uniqueCount": 1, + "fieldPath": "first_name", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "createdby", - "uniqueCount": 1, + "fieldPath": "email_address", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "createdfor", - "uniqueCount": 0, - "nullCount": 2 + "fieldPath": "priority", + "uniqueCount": 3, + "nullCount": 1, + "mean": "4.175000011920929", + "stdev": "0.4924429489953036" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7243,50 +7959,80 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 2, - "columnCount": 7, + "rowCount": 10842, + "columnCount": 13, "fieldProfiles": [ { - "fieldPath": "urn", - "uniqueCount": 1, + "fieldPath": "A", + "uniqueCount": 9661, "nullCount": 0 }, { - "fieldPath": "aspect", - "uniqueCount": 2, + "fieldPath": "B", + "uniqueCount": 35, "nullCount": 0 }, { - "fieldPath": "version", - "uniqueCount": 1, + "fieldPath": "C", + "uniqueCount": 42, "nullCount": 0 }, { - "fieldPath": "metadata", - "uniqueCount": 2, + "fieldPath": "D", + "uniqueCount": 6003, "nullCount": 0 }, { - "fieldPath": "createdon", - "uniqueCount": 1, + "fieldPath": "E", + "uniqueCount": 463, "nullCount": 0 }, { - "fieldPath": "createdby", - "uniqueCount": 1, + "fieldPath": "F", + "uniqueCount": 23, "nullCount": 0 }, { - "fieldPath": "createdfor", - "uniqueCount": 0, - "nullCount": 2 + "fieldPath": "G", + "uniqueCount": 5, + "nullCount": 0 + }, + { + "fieldPath": "H", + "uniqueCount": 94, + "nullCount": 0 + }, + { + "fieldPath": "I", + "uniqueCount": 8, + "nullCount": 0 + }, + { + "fieldPath": "J", + "uniqueCount": 121, + "nullCount": 0 + }, + { + "fieldPath": "K", + "uniqueCount": 1379, + "nullCount": 0 + }, + { + "fieldPath": "L", + "uniqueCount": 2835, + "nullCount": 0 + }, + { + "fieldPath": "M", + "uniqueCount": 35, + "nullCount": 1 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -7336,7 +8082,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -7415,7 +8161,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } } From 580390ed82e49076527ab574bf1f0c3d7dd2f4b7 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Mon, 2 Dec 2024 19:31:59 +0530 Subject: [PATCH 107/174] fix(ingest/gc): delete invalid dpis (#11998) --- .../src/datahub/ingestion/source/gc/dataprocess_cleanup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 0f35e1a67fede7..ca67cd6daa045b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -277,7 +277,12 @@ def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None: assert self.ctx.graph dpis = self.fetch_dpis(job.urn, self.config.batch_size) - dpis.sort(key=lambda x: x["created"]["time"], reverse=True) + dpis.sort( + key=lambda x: x["created"]["time"] + if x["created"] and x["created"]["time"] + else 0, + reverse=True, + ) with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor: if self.config.keep_last_n: From 411e1a30ffb7d4dd6ffbe2bd45b81b6cc5f867c7 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 2 Dec 2024 12:53:02 -0500 Subject: [PATCH 108/174] feat(airflow): show dag/task logs in CI (#11981) --- .../tests/integration/test_plugin.py | 48 +++++++++++++++++++ .../airflow-plugin/tox.ini | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py index 44efd94f834b15..3becf10703df6c 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py @@ -8,6 +8,7 @@ import random import signal import subprocess +import textwrap import time from typing import Any, Iterator, Sequence @@ -110,6 +111,48 @@ def _wait_for_dag_finish( raise NotReadyError(f"DAG has not finished yet: {dag_run['state']}") +def _dump_dag_logs(airflow_instance: AirflowInstance, dag_id: str) -> None: + # Get the dag run info + res = airflow_instance.session.get( + f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns", timeout=5 + ) + res.raise_for_status() + dag_run = res.json()["dag_runs"][0] + dag_run_id = dag_run["dag_run_id"] + + # List the tasks in the dag run + res = airflow_instance.session.get( + f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances", + timeout=5, + ) + res.raise_for_status() + task_instances = res.json()["task_instances"] + + # Sort tasks by start_date to maintain execution order + task_instances.sort(key=lambda x: x["start_date"] or "") + + print(f"\nTask execution order for DAG {dag_id}:") + for task in task_instances: + task_id = task["task_id"] + state = task["state"] + try_number = task.get("try_number", 1) + + task_header = f"Task: {task_id} (State: {state}; Try: {try_number})" + + # Get logs for the task's latest try number + try: + res = airflow_instance.session.get( + f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}" + f"/taskInstances/{task_id}/logs/{try_number}", + params={"full_content": "true"}, + timeout=5, + ) + res.raise_for_status() + print(f"\n=== {task_header} ===\n{textwrap.indent(res.text, ' ')}") + except Exception as e: + print(f"Failed to fetch logs for {task_header}: {e}") + + @contextlib.contextmanager def _run_airflow( tmp_path: pathlib.Path, @@ -377,6 +420,11 @@ def test_airflow_plugin( print("Sleeping for a few seconds to let the plugin finish...") time.sleep(10) + try: + _dump_dag_logs(airflow_instance, dag_id) + except Exception as e: + print(f"Failed to dump DAG logs: {e}") + if dag_id == DAG_TO_SKIP_INGESTION: # Verify that no MCPs were generated. assert not os.path.exists(airflow_instance.metadata_file) diff --git a/metadata-ingestion-modules/airflow-plugin/tox.ini b/metadata-ingestion-modules/airflow-plugin/tox.ini index 2e4596a24c2a6c..28c0b9532bcb8e 100644 --- a/metadata-ingestion-modules/airflow-plugin/tox.ini +++ b/metadata-ingestion-modules/airflow-plugin/tox.ini @@ -59,6 +59,6 @@ commands = [testenv:py310-airflow24] extras = dev,integration-tests,plugin-v2,test-airflow24 -[testenv:py310-airflow{26,27,28},py311-airflow{29,210}] +[testenv:py3{10,11}-airflow{26,27,28,29,210}] extras = dev,integration-tests,plugin-v2 From ce6474df5a554e2462afe8d48374d51db02dce97 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 2 Dec 2024 12:53:13 -0500 Subject: [PATCH 109/174] chore(ingest): remove deprecated calls to Urn.create_from_string (#11983) --- .../structuredproperties.py | 2 +- metadata-ingestion/src/datahub/cli/put_cli.py | 2 +- .../datahub/cli/specific/dataproduct_cli.py | 2 +- .../src/datahub/emitter/mcp_patch_builder.py | 43 ++++++++++++++++ .../source/bigquery_v2/bigquery_audit.py | 2 +- .../datahub/ingestion/source/csv_enricher.py | 2 +- .../ingestion/source/elastic_search.py | 2 +- .../source/gc/soft_deleted_entity_cleanup.py | 2 +- .../ingestion/transformer/add_dataset_tags.py | 2 +- .../transformer/generic_aspect_transformer.py | 2 +- .../datahub/integrations/assertion/common.py | 2 +- .../src/datahub/lite/duckdb_lite.py | 29 +++++------ .../src/datahub/specific/chart.py | 39 --------------- .../src/datahub/specific/dashboard.py | 39 --------------- .../src/datahub/specific/datajob.py | 50 ++----------------- .../src/datahub/utilities/urns/_urn_base.py | 2 +- .../urns/structured_properties_urn.py | 2 +- .../tests/test_helpers/mce_helpers.py | 4 +- .../unit/test_generic_aspect_transformer.py | 4 +- .../tests/unit/test_transform_dataset.py | 4 +- 20 files changed, 76 insertions(+), 160 deletions(-) diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py index 56e02e4329055a..181c70adc640a6 100644 --- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py +++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py @@ -121,7 +121,7 @@ def fqn(self) -> str: return ( self.qualified_name or self.id - or Urn.create_from_string(self.urn).get_entity_id()[0] + or Urn.from_string(self.urn).get_entity_id()[0] ) @validator("urn", pre=True, always=True) diff --git a/metadata-ingestion/src/datahub/cli/put_cli.py b/metadata-ingestion/src/datahub/cli/put_cli.py index 0a40a9f4ccf92d..d3a6fb5caaf197 100644 --- a/metadata-ingestion/src/datahub/cli/put_cli.py +++ b/metadata-ingestion/src/datahub/cli/put_cli.py @@ -105,7 +105,7 @@ def platform( """ if name.startswith(f"urn:li:{DataPlatformUrn.ENTITY_TYPE}"): - platform_urn = DataPlatformUrn.create_from_string(name) + platform_urn = DataPlatformUrn.from_string(name) platform_name = platform_urn.get_entity_id_as_string() else: platform_name = name.lower() diff --git a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py index 8ec4d3ad249376..857a6fbb4e18e5 100644 --- a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py +++ b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py @@ -45,7 +45,7 @@ def _get_owner_urn(maybe_urn: str) -> str: def _abort_if_non_existent_urn(graph: DataHubGraph, urn: str, operation: str) -> None: try: - parsed_urn: Urn = Urn.create_from_string(urn) + parsed_urn: Urn = Urn.from_string(urn) entity_type = parsed_urn.get_type() except Exception: click.secho(f"Provided urn {urn} does not seem valid", fg="red") diff --git a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py index 37903995394379..779b42e1e1ee99 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py @@ -1,4 +1,5 @@ import json +import time from collections import defaultdict from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Optional, Sequence, Union @@ -6,12 +7,15 @@ from datahub.emitter.aspect import JSON_PATCH_CONTENT_TYPE from datahub.emitter.serialization_helper import pre_json_transform from datahub.metadata.schema_classes import ( + AuditStampClass, ChangeTypeClass, + EdgeClass, GenericAspectClass, KafkaAuditHeaderClass, MetadataChangeProposalClass, SystemMetadataClass, ) +from datahub.metadata.urns import Urn from datahub.utilities.urns.urn import guess_entity_type @@ -89,3 +93,42 @@ def build(self) -> Iterable[MetadataChangeProposalClass]: ) for aspect_name, patches in self.patches.items() ] + + @classmethod + def _mint_auditstamp(cls, message: Optional[str] = None) -> AuditStampClass: + """ + Creates an AuditStampClass instance with the current timestamp and other default values. + + Args: + message: The message associated with the audit stamp (optional). + + Returns: + An instance of AuditStampClass. + """ + return AuditStampClass( + time=int(time.time() * 1000.0), + actor="urn:li:corpuser:datahub", + message=message, + ) + + @classmethod + def _ensure_urn_type( + cls, entity_type: str, edges: List[EdgeClass], context: str + ) -> None: + """ + Ensures that the destination URNs in the given edges have the specified entity type. + + Args: + entity_type: The entity type to check against. + edges: A list of Edge objects. + context: The context or description of the operation. + + Raises: + ValueError: If any of the destination URNs is not of the specified entity type. + """ + for e in edges: + urn = Urn.from_string(e.destinationUrn) + if not urn.entity_type == entity_type: + raise ValueError( + f"{context}: {e.destinationUrn} is not of type {entity_type}" + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py index 319c838d2658ad..42f82704c81b99 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py @@ -190,7 +190,7 @@ def from_string_name(cls, ref: str) -> "BigQueryTableRef": @classmethod def from_urn(cls, urn: str) -> "BigQueryTableRef": """Raises: ValueError if urn is not a valid BigQuery table URN.""" - dataset_urn = DatasetUrn.create_from_string(urn) + dataset_urn = DatasetUrn.from_string(urn) split = dataset_urn.name.rsplit(".", 3) if len(split) == 3: project, dataset, table = split diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py index e4829f8713cf76..42e025073b534e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py +++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py @@ -653,7 +653,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: is_resource_row: bool = not row["subresource"] entity_urn = row["resource"] - entity_type = Urn.create_from_string(row["resource"]).get_type() + entity_type = Urn.from_string(row["resource"]).get_type() term_associations: List[ GlossaryTermAssociationClass diff --git a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py index aa5913f5dc66b1..99aa5f54f6a576 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py +++ b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py @@ -227,7 +227,7 @@ def collapse_name(name: str, collapse_urns: CollapseUrns) -> str: def collapse_urn(urn: str, collapse_urns: CollapseUrns) -> str: if len(collapse_urns.urns_suffix_regex) == 0: return urn - urn_obj = DatasetUrn.create_from_string(urn) + urn_obj = DatasetUrn.from_string(urn) name = collapse_name(name=urn_obj.get_dataset_name(), collapse_urns=collapse_urns) data_platform_urn = urn_obj.get_data_platform_urn() return str( diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py index 7ec7bb7e589d63..3b367cdea58134 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py @@ -104,7 +104,7 @@ def __init__( def delete_entity(self, urn: str) -> None: assert self.ctx.graph - entity_urn = Urn.create_from_string(urn) + entity_urn = Urn.from_string(urn) self.report.num_soft_deleted_entity_removed += 1 self.report.num_soft_deleted_entity_removed_by_type[entity_urn.entity_type] = ( self.report.num_soft_deleted_entity_removed_by_type.get( diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py index c60f4dca28882d..355ca7a373653f 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py @@ -74,7 +74,7 @@ def handle_end_of_stream( logger.debug("Generating tags") for tag_association in self.processed_tags.values(): - tag_urn = TagUrn.create_from_string(tag_association.tag) + tag_urn = TagUrn.from_string(tag_association.tag) mcps.append( MetadataChangeProposalWrapper( entityUrn=tag_urn.urn(), diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py index 3c0bc00633e7cb..5bf70274dce89d 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py @@ -100,7 +100,7 @@ def transform( ) if transformed_aspect: # for end of stream records, we modify the workunit-id - structured_urn = Urn.create_from_string(urn) + structured_urn = Urn.from_string(urn) simple_name = "-".join(structured_urn.get_entity_id()) record_metadata = envelope.metadata.copy() record_metadata.update( diff --git a/metadata-ingestion/src/datahub/integrations/assertion/common.py b/metadata-ingestion/src/datahub/integrations/assertion/common.py index 9ffad5cf66640a..f0b7510df14a35 100644 --- a/metadata-ingestion/src/datahub/integrations/assertion/common.py +++ b/metadata-ingestion/src/datahub/integrations/assertion/common.py @@ -42,7 +42,7 @@ def get_entity_name(assertion: BaseEntityAssertion) -> Tuple[str, str, str]: if qualified_name is not None: parts = qualified_name.split(".") else: - urn_id = Urn.create_from_string(assertion.entity).entity_ids[1] + urn_id = Urn.from_string(assertion.entity).entity_ids[1] parts = urn_id.split(".") if len(parts) > 3: parts = parts[-3:] diff --git a/metadata-ingestion/src/datahub/lite/duckdb_lite.py b/metadata-ingestion/src/datahub/lite/duckdb_lite.py index f40a2e9498e62c..89317383520923 100644 --- a/metadata-ingestion/src/datahub/lite/duckdb_lite.py +++ b/metadata-ingestion/src/datahub/lite/duckdb_lite.py @@ -609,7 +609,7 @@ def get_typed_aspect( aspect_map, DataPlatformInstanceClass ) # type: ignore - needs_platform = Urn.create_from_string(entity_urn).get_type() in [ + needs_platform = Urn.from_string(entity_urn).get_type() in [ "dataset", "container", "chart", @@ -617,7 +617,7 @@ def get_typed_aspect( "dataFlow", "dataJob", ] - entity_urn_parsed = Urn.create_from_string(entity_urn) + entity_urn_parsed = Urn.from_string(entity_urn) if entity_urn_parsed.get_type() in ["dataFlow", "dataJob"]: self.add_edge( entity_urn, @@ -630,15 +630,12 @@ def get_typed_aspect( # this is a top-level entity if not dpi: logger.debug(f"No data platform instance for {entity_urn}") - maybe_parent_urn = Urn.create_from_string(entity_urn).get_entity_id()[0] + maybe_parent_urn = Urn.from_string(entity_urn).get_entity_id()[0] needs_dpi = False if maybe_parent_urn.startswith(Urn.URN_PREFIX): parent_urn = maybe_parent_urn - if ( - Urn.create_from_string(maybe_parent_urn).get_type() - == "dataPlatform" - ): - data_platform_urn = DataPlatformUrn.create_from_string( + if Urn.from_string(maybe_parent_urn).get_type() == "dataPlatform": + data_platform_urn = DataPlatformUrn.from_string( maybe_parent_urn ) needs_dpi = True @@ -660,7 +657,7 @@ def get_typed_aspect( logger.error(f"Failed to generate edges entity {entity_urn}", e) parent_urn = str(data_platform_instance_urn) else: - data_platform_urn = DataPlatformUrn.create_from_string(dpi.platform) + data_platform_urn = DataPlatformUrn.from_string(dpi.platform) data_platform_instance = dpi.instance or "default" data_platform_instance_urn = Urn( entity_type="dataPlatformInstance", @@ -673,9 +670,7 @@ def get_typed_aspect( parent_urn = "__root__" types = ( - subtypes.typeNames - if subtypes - else [Urn.create_from_string(entity_urn).get_type()] + subtypes.typeNames if subtypes else [Urn.from_string(entity_urn).get_type()] ) for t in types: type_urn = Urn(entity_type="systemNode", entity_id=[parent_urn, t]) @@ -686,7 +681,7 @@ def get_typed_aspect( def _create_edges_from_data_platform_instance( self, data_platform_instance_urn: Urn ) -> None: - data_platform_urn = DataPlatformUrn.create_from_string( + data_platform_urn = DataPlatformUrn.from_string( data_platform_instance_urn.get_entity_id()[0] ) data_platform_instances_urn = Urn( @@ -735,7 +730,7 @@ def post_update_hook( if isinstance(aspect, DatasetPropertiesClass): dp: DatasetPropertiesClass = aspect if dp.name: - specific_urn = DatasetUrn.create_from_string(entity_urn) + specific_urn = DatasetUrn.from_string(entity_urn) if ( specific_urn.get_data_platform_urn().get_entity_id_as_string() == "looker" @@ -755,7 +750,7 @@ def post_update_hook( self.add_edge(entity_urn, "name", cp.name, remove_existing=True) elif isinstance(aspect, DataPlatformInstanceClass): dpi: DataPlatformInstanceClass = aspect - data_platform_urn = DataPlatformUrn.create_from_string(dpi.platform) + data_platform_urn = DataPlatformUrn.from_string(dpi.platform) data_platform_instance = dpi.instance or "default" data_platform_instance_urn = Urn( entity_type="dataPlatformInstance", @@ -763,7 +758,7 @@ def post_update_hook( ) self._create_edges_from_data_platform_instance(data_platform_instance_urn) elif isinstance(aspect, ChartInfoClass): - urn = Urn.create_from_string(entity_urn) + urn = Urn.from_string(entity_urn) self.add_edge( entity_urn, "name", @@ -771,7 +766,7 @@ def post_update_hook( remove_existing=True, ) elif isinstance(aspect, DashboardInfoClass): - urn = Urn.create_from_string(entity_urn) + urn = Urn.from_string(entity_urn) self.add_edge( entity_urn, "name", diff --git a/metadata-ingestion/src/datahub/specific/chart.py b/metadata-ingestion/src/datahub/specific/chart.py index cc68168b68db7e..104a7c21a07e2f 100644 --- a/metadata-ingestion/src/datahub/specific/chart.py +++ b/metadata-ingestion/src/datahub/specific/chart.py @@ -1,10 +1,8 @@ -import time from typing import Dict, List, Optional, Union from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( AccessLevelClass, - AuditStampClass, ChangeAuditStampsClass, ChartInfoClass as ChartInfo, ChartTypeClass, @@ -47,43 +45,6 @@ def __init__( ) self.ownership_patch_helper = OwnershipPatchHelper(self) - def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass: - """ - Creates an AuditStampClass instance with the current timestamp and other default values. - - Args: - message: The message associated with the audit stamp (optional). - - Returns: - An instance of AuditStampClass. - """ - return AuditStampClass( - time=int(time.time() * 1000.0), - actor="urn:li:corpuser:datahub", - message=message, - ) - - def _ensure_urn_type( - self, entity_type: str, edges: List[Edge], context: str - ) -> None: - """ - Ensures that the destination URNs in the given edges have the specified entity type. - - Args: - entity_type: The entity type to check against. - edges: A list of Edge objects. - context: The context or description of the operation. - - Raises: - ValueError: If any of the destination URNs is not of the specified entity type. - """ - for e in edges: - urn = Urn.create_from_string(e.destinationUrn) - if not urn.get_type() == entity_type: - raise ValueError( - f"{context}: {e.destinationUrn} is not of type {entity_type}" - ) - def add_owner(self, owner: Owner) -> "ChartPatchBuilder": """ Adds an owner to the ChartPatchBuilder. diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index f57df15914369c..da5abbfd1dc129 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -1,10 +1,8 @@ -import time from typing import Dict, List, Optional, Union from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( AccessLevelClass, - AuditStampClass, ChangeAuditStampsClass, DashboardInfoClass as DashboardInfo, EdgeClass as Edge, @@ -46,43 +44,6 @@ def __init__( ) self.ownership_patch_helper = OwnershipPatchHelper(self) - def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass: - """ - Creates an AuditStampClass instance with the current timestamp and other default values. - - Args: - message: The message associated with the audit stamp (optional). - - Returns: - An instance of AuditStampClass. - """ - return AuditStampClass( - time=int(time.time() * 1000.0), - actor="urn:li:corpuser:datahub", - message=message, - ) - - def _ensure_urn_type( - self, entity_type: str, edges: List[Edge], context: str - ) -> None: - """ - Ensures that the destination URNs in the given edges have the specified entity type. - - Args: - entity_type: The entity type to check against. - edges: A list of Edge objects. - context: The context or description of the operation. - - Raises: - ValueError: If any of the destination URNs is not of the specified entity type. - """ - for e in edges: - urn = Urn.create_from_string(e.destinationUrn) - if not urn.get_type() == entity_type: - raise ValueError( - f"{context}: {e.destinationUrn} is not of type {entity_type}" - ) - def add_owner(self, owner: Owner) -> "DashboardPatchBuilder": """ Adds an owner to the DashboardPatchBuilder. diff --git a/metadata-ingestion/src/datahub/specific/datajob.py b/metadata-ingestion/src/datahub/specific/datajob.py index 8da8edc8ef0f22..fb7b0ae7816f17 100644 --- a/metadata-ingestion/src/datahub/specific/datajob.py +++ b/metadata-ingestion/src/datahub/specific/datajob.py @@ -1,9 +1,7 @@ -import time from typing import Dict, List, Optional, Union from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( - AuditStampClass, DataJobInfoClass as DataJobInfo, DataJobInputOutputClass as DataJobInputOutput, EdgeClass as Edge, @@ -16,10 +14,9 @@ SystemMetadataClass, TagAssociationClass as Tag, ) +from datahub.metadata.urns import SchemaFieldUrn, TagUrn, Urn from datahub.specific.custom_properties import CustomPropertiesPatchHelper from datahub.specific.ownership import OwnershipPatchHelper -from datahub.utilities.urns.tag_urn import TagUrn -from datahub.utilities.urns.urn import Urn class DataJobPatchBuilder(MetadataPatchProposal): @@ -45,43 +42,6 @@ def __init__( ) self.ownership_patch_helper = OwnershipPatchHelper(self) - def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass: - """ - Creates an AuditStampClass instance with the current timestamp and other default values. - - Args: - message: The message associated with the audit stamp (optional). - - Returns: - An instance of AuditStampClass. - """ - return AuditStampClass( - time=int(time.time() * 1000.0), - actor="urn:li:corpuser:datahub", - message=message, - ) - - def _ensure_urn_type( - self, entity_type: str, edges: List[Edge], context: str - ) -> None: - """ - Ensures that the destination URNs in the given edges have the specified entity type. - - Args: - entity_type: The entity type to check against. - edges: A list of Edge objects. - context: The context or description of the operation. - - Raises: - ValueError: If any of the destination URNs is not of the specified entity type. - """ - for e in edges: - urn = Urn.create_from_string(e.destinationUrn) - if not urn.get_type() == entity_type: - raise ValueError( - f"{context}: {e.destinationUrn} is not of type {entity_type}" - ) - def add_owner(self, owner: Owner) -> "DataJobPatchBuilder": """ Adds an owner to the DataJobPatchBuilder. @@ -392,9 +352,7 @@ def add_input_dataset_field(self, input: Union[Urn, str]) -> "DataJobPatchBuilde ValueError: If the input is not a Schema Field urn. """ input_urn = str(input) - urn = Urn.create_from_string(input_urn) - if not urn.get_type() == "schemaField": - raise ValueError(f"Input {input} is not a Schema Field urn") + assert SchemaFieldUrn.from_string(input_urn) self._add_patch( DataJobInputOutput.ASPECT_NAME, @@ -466,9 +424,7 @@ def add_output_dataset_field( ValueError: If the output is not a Schema Field urn. """ output_urn = str(output) - urn = Urn.create_from_string(output_urn) - if not urn.get_type() == "schemaField": - raise ValueError(f"Input {output} is not a Schema Field urn") + assert SchemaFieldUrn.from_string(output_urn) self._add_patch( DataJobInputOutput.ASPECT_NAME, diff --git a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py index 1b50d4b2fe810c..7dadd16fb7f1c2 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py +++ b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py @@ -200,7 +200,7 @@ def get_entity_id_as_string(self) -> str: @classmethod @deprecated(reason="no longer needed") def validate(cls, urn_str: str) -> None: - Urn.create_from_string(urn_str) + Urn.from_string(urn_str) @staticmethod def url_encode(urn: str) -> str: diff --git a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py index 6774978c7a76d9..748d5519f14773 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py @@ -4,4 +4,4 @@ def make_structured_property_urn(structured_property_id: str) -> str: - return str(StructuredPropertyUrn.create_from_string(structured_property_id)) + return str(StructuredPropertyUrn.from_string(structured_property_id)) diff --git a/metadata-ingestion/tests/test_helpers/mce_helpers.py b/metadata-ingestion/tests/test_helpers/mce_helpers.py index f4c629df7dba4e..0105e6d596970b 100644 --- a/metadata-ingestion/tests/test_helpers/mce_helpers.py +++ b/metadata-ingestion/tests/test_helpers/mce_helpers.py @@ -323,7 +323,7 @@ def assert_entity_mce_aspect( ) -> int: # TODO: Replace with read_metadata_file() test_output = load_json_file(file) - entity_type = Urn.create_from_string(entity_urn).get_type() + entity_type = Urn.from_string(entity_urn).get_type() assert isinstance(test_output, list) # mce urns mces: List[MetadataChangeEventClass] = [ @@ -346,7 +346,7 @@ def assert_entity_mcp_aspect( ) -> int: # TODO: Replace with read_metadata_file() test_output = load_json_file(file) - entity_type = Urn.create_from_string(entity_urn).get_type() + entity_type = Urn.from_string(entity_urn).get_type() assert isinstance(test_output, list) # mcps that match entity_urn mcps: List[MetadataChangeProposalWrapper] = [ diff --git a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py index 18b0d9fd400412..52d7aa7f509c9e 100644 --- a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py +++ b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py @@ -51,7 +51,7 @@ def make_mcpw( ) -> MetadataChangeProposalWrapper: return MetadataChangeProposalWrapper( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, @@ -65,7 +65,7 @@ def make_mcpc( ) -> MetadataChangeProposalClass: return MetadataChangeProposalClass( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index 389f7b70b3311e..5151be9c8b1997 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -122,7 +122,7 @@ def make_generic_dataset_mcp( ) -> MetadataChangeProposalWrapper: return MetadataChangeProposalWrapper( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, @@ -138,7 +138,7 @@ def make_generic_container_mcp( aspect = models.StatusClass(removed=False) return MetadataChangeProposalWrapper( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, From 59181e4637af3308ac5ee18a54a2a7f0137d05d9 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 2 Dec 2024 23:36:51 +0530 Subject: [PATCH 110/174] fix(ingest): resolve missing numeric types for profiling (#11991) --- .../ingestion/source/ge_data_profiler.py | 24 ++++++++++++++++++- .../datahub/ingestion/source/sql/sql_types.py | 16 +++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index f7d783cd3dec0b..c2c43a0e805b24 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -57,7 +57,11 @@ convert_to_cardinality, ) from datahub.ingestion.source.sql.sql_report import SQLSourceReport -from datahub.metadata.com.linkedin.pegasus2avro.schema import EditableSchemaMetadata +from datahub.ingestion.source.sql.sql_types import resolve_sql_type +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + EditableSchemaMetadata, + NumberType, +) from datahub.metadata.schema_classes import ( DatasetFieldProfileClass, DatasetProfileClass, @@ -361,6 +365,8 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase): platform: str env: str + column_types: Dict[str, str] = dataclasses.field(default_factory=dict) + def _get_columns_to_profile(self) -> List[str]: if not self.config.any_field_level_metrics_enabled(): return [] @@ -374,6 +380,7 @@ def _get_columns_to_profile(self) -> List[str]: for col_dict in self.dataset.columns: col = col_dict["name"] + self.column_types[col] = str(col_dict["type"]) # We expect the allow/deny patterns to specify '.' if not self.config._allow_deny_patterns.allowed( f"{self.dataset_name}.{col}" @@ -430,6 +437,21 @@ def _get_column_type(self, column_spec: _SingleColumnSpec, column: str) -> None: self.dataset, column ) + if column_spec.type_ == ProfilerDataType.UNKNOWN: + try: + datahub_field_type = resolve_sql_type( + self.column_types[column], self.dataset.engine.dialect.name.lower() + ) + except Exception as e: + logger.debug( + f"Error resolving sql type {self.column_types[column]}: {e}" + ) + datahub_field_type = None + if datahub_field_type is None: + return + if isinstance(datahub_field_type, NumberType): + column_spec.type_ = ProfilerDataType.NUMERIC + @_run_with_query_combiner def _get_column_cardinality( self, column_spec: _SingleColumnSpec, column: str diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py index 89ca160ba1f487..9ec73a9af96dc5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py @@ -276,7 +276,6 @@ def resolve_vertica_modified_type(type_string: str) -> Any: return VERTICA_SQL_TYPES_MAP[type_string] -# see https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html SNOWFLAKE_TYPES_MAP: Dict[str, Any] = { "NUMBER": NumberType, "DECIMAL": NumberType, @@ -312,6 +311,18 @@ def resolve_vertica_modified_type(type_string: str) -> Any: "GEOGRAPHY": None, } + +def resolve_snowflake_modified_type(type_string: str) -> Any: + # Match types with precision and scale, e.g., 'DECIMAL(38,0)' + match = re.match(r"([a-zA-Z_]+)\(\d+,\s\d+\)", type_string) + if match: + modified_type_base = match.group(1) # Extract the base type + return SNOWFLAKE_TYPES_MAP.get(modified_type_base, None) + + # Fallback for types without precision/scale + return SNOWFLAKE_TYPES_MAP.get(type_string, None) + + # see https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/sqlalchemy_bigquery/_types.py#L32 BIGQUERY_TYPES_MAP: Dict[str, Any] = { "STRING": StringType, @@ -380,6 +391,7 @@ def resolve_vertica_modified_type(type_string: str) -> Any: "row": RecordType, "map": MapType, "array": ArrayType, + "json": RecordType, } # https://docs.aws.amazon.com/athena/latest/ug/data-types.html @@ -490,7 +502,7 @@ def resolve_sql_type( TypeClass = resolve_vertica_modified_type(column_type) elif platform == "snowflake": # Snowflake types are uppercase, so we check that. - TypeClass = _merged_mapping.get(column_type.upper()) + TypeClass = resolve_snowflake_modified_type(column_type.upper()) if TypeClass: return TypeClass() From 2f20c529f4306c4dccedb13a4be461f06d62e572 Mon Sep 17 00:00:00 2001 From: Jonny Dixon <45681293+acrylJonny@users.noreply.github.com> Date: Mon, 2 Dec 2024 18:08:20 +0000 Subject: [PATCH 111/174] fix(docs): Add spark.datahub.stage_metadata_coalescing to recommended configuration for databricks (#11800) --- metadata-integration/java/acryl-spark-lineage/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metadata-integration/java/acryl-spark-lineage/README.md b/metadata-integration/java/acryl-spark-lineage/README.md index 267e979b0fa073..97851e90e860ed 100644 --- a/metadata-integration/java/acryl-spark-lineage/README.md +++ b/metadata-integration/java/acryl-spark-lineage/README.md @@ -125,9 +125,10 @@ information like tokens. configurations under `Spark Config`. ```text - spark.extraListeners datahub.spark.DatahubSparkListener - spark.datahub.rest.server http://localhost:8080 - spark.datahub.databricks.cluster cluster-name + spark.extraListeners datahub.spark.DatahubSparkListener + spark.datahub.rest.server http://localhost:8080 + spark.datahub.stage_metadata_coalescing true + spark.datahub.databricks.cluster cluster-name ``` - Click the **Init Scripts** tab. Set cluster init script as `dbfs:/datahub/init.sh`. From 85b42e3ea5c7ef809252a2d94b7fb0915243e14d Mon Sep 17 00:00:00 2001 From: Chakru <161002324+chakru-r@users.noreply.github.com> Date: Tue, 3 Dec 2024 06:57:43 +0530 Subject: [PATCH 112/174] build(coverage): enable code coverage for java and python (#11992) Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- datahub-graphql-core/build.gradle | 1 + datahub-upgrade/build.gradle | 1 + entity-registry/build.gradle | 3 ++ .../template/UpstreamLineageTemplateTest.java | 4 +-- gradle/coverage/java-coverage.gradle | 31 +++++++++++++++++++ gradle/coverage/python-coverage.gradle | 18 +++++++++++ ingestion-scheduler/build.gradle | 2 ++ li-utils/build.gradle | 3 ++ metadata-auth/auth-api/build.gradle | 1 + metadata-dao-impl/kafka-producer/build.gradle | 2 ++ metadata-events/mxe-utils-avro/build.gradle | 2 ++ .../airflow-plugin/build.gradle | 4 ++- .../dagster-plugin/build.gradle | 4 ++- .../gx-plugin/build.gradle | 4 ++- .../prefect-plugin/build.gradle | 6 ++-- metadata-ingestion/build.gradle | 14 ++++----- .../java/acryl-spark-lineage/build.gradle | 6 +--- .../java/datahub-client/build.gradle | 6 +--- .../java/datahub-event/build.gradle | 7 +---- .../java/datahub-protobuf/build.gradle | 11 ++----- .../java/openlineage-converter/build.gradle | 10 ++---- .../java/spark-lineage-legacy/build.gradle | 11 ------- metadata-io/build.gradle | 2 ++ metadata-io/metadata-io-api/build.gradle | 2 ++ metadata-jobs/common/build.gradle | 2 ++ metadata-jobs/mae-consumer-job/build.gradle | 1 + metadata-jobs/mae-consumer/build.gradle | 2 +- metadata-jobs/mce-consumer-job/build.gradle | 1 + metadata-jobs/mce-consumer/build.gradle | 1 + metadata-jobs/pe-consumer/build.gradle | 2 ++ metadata-models-custom/build.gradle | 2 ++ metadata-models-validator/build.gradle | 2 ++ metadata-models/build.gradle | 1 + metadata-operation-context/build.gradle | 1 + metadata-service/auth-config/build.gradle | 2 ++ metadata-service/auth-filter/build.gradle | 2 ++ metadata-service/auth-impl/build.gradle | 1 + .../auth-servlet-impl/build.gradle | 2 ++ metadata-service/configuration/build.gradle | 2 ++ metadata-service/factories/build.gradle | 1 + .../graphql-servlet-impl/build.gradle | 2 ++ .../openapi-analytics-servlet/build.gradle | 2 ++ .../openapi-entity-servlet/build.gradle | 2 ++ metadata-service/openapi-servlet/build.gradle | 2 ++ metadata-service/plugin/build.gradle | 2 ++ metadata-service/restli-api/build.gradle | 2 ++ .../restli-client-api/build.gradle | 2 ++ metadata-service/restli-client/build.gradle | 2 ++ .../restli-servlet-impl/build.gradle | 2 ++ .../schema-registry-api/build.gradle | 1 + .../schema-registry-servlet/build.gradle | 2 ++ metadata-service/services/build.gradle | 2 ++ metadata-service/servlet/build.gradle | 2 ++ metadata-service/war/build.gradle | 1 + metadata-utils/build.gradle | 2 ++ mock-entity-registry/build.gradle | 2 ++ 56 files changed, 151 insertions(+), 59 deletions(-) create mode 100644 gradle/coverage/java-coverage.gradle create mode 100644 gradle/coverage/python-coverage.gradle diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index 49a7fa7fbfbc2f..47ada8e9929dd3 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -3,6 +3,7 @@ plugins { id "io.github.kobylynskyi.graphql.codegen" version "4.1.1" } +apply from: '../gradle/coverage/java-coverage.gradle' dependencies { implementation project(':metadata-service:restli-client-api') diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle index b783efa09713d1..372b0eb0570b98 100644 --- a/datahub-upgrade/build.gradle +++ b/datahub-upgrade/build.gradle @@ -5,6 +5,7 @@ plugins { } apply from: "../gradle/versioning/versioning.gradle" +apply from: "../gradle/coverage/java-coverage.gradle" ext { docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle index 22e5b601d39db2..e5baa95967f304 100644 --- a/entity-registry/build.gradle +++ b/entity-registry/build.gradle @@ -4,6 +4,8 @@ plugins { id 'java-test-fixtures' } +apply from: "../gradle/coverage/java-coverage.gradle" + dependencies { implementation spec.product.pegasus.data implementation spec.product.pegasus.generator @@ -53,3 +55,4 @@ dependencies { testFixturesAnnotationProcessor externalDependency.lombok } compileTestJava.dependsOn tasks.getByPath(':entity-registry:custom-test-model:modelDeploy') + diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java index f9af15a3d4cc6c..f934dd8961ca37 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java @@ -272,8 +272,8 @@ public void testLargePatchStandard() throws Exception { UpstreamLineage result = upstreamLineageTemplate.applyPatch(upstreamLineage, jsonPatch); long end = System.currentTimeMillis(); assertTrue( - end - start < 10000, - String.format("Expected less then 10 seconds patch actual %s ms", end - start)); + end - start < 20000, + String.format("Expected less then 20 seconds patch actual %s ms", end - start)); assertEquals(result.getUpstreams().size(), 187, "Expected 1 less upstream"); assertEquals(result.getFineGrainedLineages().size(), 607); diff --git a/gradle/coverage/java-coverage.gradle b/gradle/coverage/java-coverage.gradle new file mode 100644 index 00000000000000..17260c1a309788 --- /dev/null +++ b/gradle/coverage/java-coverage.gradle @@ -0,0 +1,31 @@ +apply plugin: "jacoco" + +jacoco { + toolVersion = "0.8.12" +} + +/* +These need to run after evaluation since jacoco plugin alters the test task based on the included test +lib dependencies defined in each subproject (junit or testNG) +*/ +afterEvaluate { + test { + finalizedBy jacocoTestReport + } + + jacocoTestReport { + dependsOn test + reports { + xml { + required = true + /* + Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one + folder will minimize the time spent searching through the full source tree. + */ + outputLocation = rootProject.layout.buildDirectory.file("coverage-reports/jacoco-${project.name}.xml") + } + csv.required = false + html.required = false + } + } +} \ No newline at end of file diff --git a/gradle/coverage/python-coverage.gradle b/gradle/coverage/python-coverage.gradle new file mode 100644 index 00000000000000..0ab921dfb21ffd --- /dev/null +++ b/gradle/coverage/python-coverage.gradle @@ -0,0 +1,18 @@ +//coverage related args to be passed to pytest +ext.get_coverage_args = { test_name = "" -> + + def coverage_file_name = "pycov-${project.name}${test_name}.xml" + + /* + Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one folder + will minimize the time spent searching through the full source tree. + */ + def base_path = "${rootProject.buildDir}/coverage-reports" + + /* + --cov=src was added via setup.cfg in many of the python projects but for some reason, was not getting picked up + consistently, so adding it explicitly. Centralizing these params would make it easier to adjust them for all python + projects (with overrides being in the sub-project build script.) + */ + return "--cov-report xml:${base_path}/${coverage_file_name} --cov=src" +} \ No newline at end of file diff --git a/ingestion-scheduler/build.gradle b/ingestion-scheduler/build.gradle index 9505ec57aa858c..fb1cfd4e27a4b5 100644 --- a/ingestion-scheduler/build.gradle +++ b/ingestion-scheduler/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: "../gradle/coverage/java-coverage.gradle" + dependencies { implementation project(path: ':metadata-models') implementation project(path: ':metadata-io') diff --git a/li-utils/build.gradle b/li-utils/build.gradle index 975cd2bccccf31..e00fcc28546123 100644 --- a/li-utils/build.gradle +++ b/li-utils/build.gradle @@ -1,8 +1,11 @@ + plugins { id 'java-library' id 'pegasus' } +apply from: "../gradle/coverage/java-coverage.gradle" + dependencies { api spec.product.pegasus.data diff --git a/metadata-auth/auth-api/build.gradle b/metadata-auth/auth-api/build.gradle index acc1af7e2e3ad6..3aafaf45bc2cb8 100644 --- a/metadata-auth/auth-api/build.gradle +++ b/metadata-auth/auth-api/build.gradle @@ -8,6 +8,7 @@ apply plugin: 'signing' apply plugin: 'maven-publish' apply plugin: 'io.codearte.nexus-staging' apply from: '../../metadata-integration/java/versioning.gradle' +apply from: '../../gradle/coverage/java-coverage.gradle' jar { archiveClassifier = "lib" diff --git a/metadata-dao-impl/kafka-producer/build.gradle b/metadata-dao-impl/kafka-producer/build.gradle index 2df15309810dba..d3ba031817afe8 100644 --- a/metadata-dao-impl/kafka-producer/build.gradle +++ b/metadata-dao-impl/kafka-producer/build.gradle @@ -1,5 +1,7 @@ apply plugin: 'java' +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-events:mxe-avro') implementation project(':metadata-events:mxe-registration') diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle index 860ced6af25813..f893fadb8e2dff 100644 --- a/metadata-events/mxe-utils-avro/build.gradle +++ b/metadata-events/mxe-utils-avro/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: "../../gradle/coverage/java-coverage.gradle" + dependencies { api project(':metadata-events:mxe-avro') api project(':metadata-models') diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index 9506609a10044a..f30858ba6a14ef 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -97,7 +99,7 @@ task testQuick(type: Exec, dependsOn: installTest) { inputs.files(project.fileTree(dir: "tests/")) commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" } diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle index 0f11af9ca83d73..0d57bb5bfdff70 100644 --- a/metadata-ingestion-modules/dagster-plugin/build.gradle +++ b/metadata-ingestion-modules/dagster-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) { outputs.dir("${venv_name}") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml" } task buildWheel(type: Exec, dependsOn: [environmentSetup]) { diff --git a/metadata-ingestion-modules/gx-plugin/build.gradle b/metadata-ingestion-modules/gx-plugin/build.gradle index 2288ae6bd83961..70bc3e0360a3a4 100644 --- a/metadata-ingestion-modules/gx-plugin/build.gradle +++ b/metadata-ingestion-modules/gx-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) { outputs.dir("${venv_name}") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml" } task buildWheel(type: Exec, dependsOn: [environmentSetup]) { diff --git a/metadata-ingestion-modules/prefect-plugin/build.gradle b/metadata-ingestion-modules/prefect-plugin/build.gradle index bc091c64933645..a2b240cc8bf708 100644 --- a/metadata-ingestion-modules/prefect-plugin/build.gradle +++ b/metadata-ingestion-modules/prefect-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -82,14 +84,14 @@ task testQuick(type: Exec, dependsOn: installDevTest) { outputs.dir("${venv_name}") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s" + "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s" } task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" + "pytest -m 'not slow_integration' -vv ${get_coverage_args('full')} --continue-on-collection-errors --junit-xml=junit.full.xml" } diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index 4e03dd6e2faaf2..2c5d8e6c9646a8 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -11,10 +13,6 @@ if (!project.hasProperty("extra_pip_requirements")) { ext.extra_pip_requirements = "" } -def get_coverage_arg(test_name) { - return "--cov-report xml:coverage_${test_name}.xml " -} - task checkPythonVersion(type: Exec) { commandLine python_executable, '-c', 'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info[:2]} not allowed"' @@ -134,7 +132,7 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") - def cvg_arg = get_coverage_arg("quick") + def cvg_arg = get_coverage_args("quick") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} tests/unit --random-order --durations=20 -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" @@ -166,19 +164,19 @@ task testSingle(dependsOn: [installDevTest]) { } task testIntegrationBatch0(type: Exec, dependsOn: [installDevTest]) { - def cvg_arg = get_coverage_arg("intBatch0") + def cvg_arg = get_coverage_args("intBatch0") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} --durations=50 -m 'integration_batch_0' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch0.xml" } task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) { - def cvg_arg = get_coverage_arg("intBatch1") + def cvg_arg = get_coverage_args("intBatch1") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml" } task testIntegrationBatch2(type: Exec, dependsOn: [installDevTest]) { - def cvg_arg = get_coverage_arg("intBatch2") + def cvg_arg = get_coverage_args("intBatch2") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} --durations=20 -m 'integration_batch_2' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch2.xml" diff --git a/metadata-integration/java/acryl-spark-lineage/build.gradle b/metadata-integration/java/acryl-spark-lineage/build.gradle index 3f83e5657bbf4d..940ebb98485367 100644 --- a/metadata-integration/java/acryl-spark-lineage/build.gradle +++ b/metadata-integration/java/acryl-spark-lineage/build.gradle @@ -6,7 +6,7 @@ apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' -apply plugin: 'jacoco' +apply from: '../../../gradle/coverage/java-coverage.gradle' apply from: '../versioning.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -163,14 +163,10 @@ checkShadowJar { dependsOn shadowJar } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} test { forkEvery = 1 useJUnit() - finalizedBy jacocoTestReport } assemble { diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 56a486ad043305..87a9623bbf062f 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -2,12 +2,12 @@ plugins { id("com.palantir.git-version") apply false id 'java-library' id 'com.github.johnrengelman.shadow' - id 'jacoco' id 'signing' id 'io.codearte.nexus-staging' id 'maven-publish' } +apply from: "../../../gradle/coverage/java-coverage.gradle" apply from: "../versioning.gradle" import org.apache.tools.ant.filters.ReplaceTokens @@ -59,15 +59,11 @@ task copyAvroSchemas { compileJava.dependsOn copyAvroSchemas -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} test { // to avoid simultaneous executions of tests when complete build is run mustRunAfter(":metadata-io:test") useJUnit() - finalizedBy jacocoTestReport } task checkShadowJar(type: Exec) { diff --git a/metadata-integration/java/datahub-event/build.gradle b/metadata-integration/java/datahub-event/build.gradle index 395065404d1db8..24e119c6229369 100644 --- a/metadata-integration/java/datahub-event/build.gradle +++ b/metadata-integration/java/datahub-event/build.gradle @@ -2,12 +2,12 @@ plugins { id("com.palantir.git-version") apply false id 'java' id 'com.github.johnrengelman.shadow' - id 'jacoco' id 'signing' id 'io.codearte.nexus-staging' id 'maven-publish' } +apply from: "../../../gradle/coverage/java-coverage.gradle" apply from: "../versioning.gradle" dependencies { @@ -29,10 +29,6 @@ dependencies { testRuntimeOnly externalDependency.logbackClassicJava8 } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - task copyAvroSchemas { dependsOn(':metadata-events:mxe-schemas:renameNamespace') copy { @@ -47,7 +43,6 @@ test { // to avoid simultaneous executions of tests when complete build is run mustRunAfter(":metadata-io:test") useJUnit() - finalizedBy jacocoTestReport } // task sourcesJar(type: Jar) { diff --git a/metadata-integration/java/datahub-protobuf/build.gradle b/metadata-integration/java/datahub-protobuf/build.gradle index 8bc71c5da8f4a3..97595fd1345dc7 100644 --- a/metadata-integration/java/datahub-protobuf/build.gradle +++ b/metadata-integration/java/datahub-protobuf/build.gradle @@ -3,12 +3,12 @@ plugins { id "application" } apply plugin: 'java' -apply plugin: 'jacoco' apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' apply from: '../versioning.gradle' +apply from: '../../../gradle/coverage/java-coverage.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -57,13 +57,6 @@ task compileProtobuf { } } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - -test.finalizedBy jacocoTestReport - - task checkShadowJar(type: Exec) { commandLine 'sh', '-c', 'scripts/check_jar.sh' } @@ -202,4 +195,4 @@ nexusStaging { password = System.getenv("NEXUS_PASSWORD") } -startScripts.dependsOn shadowJar \ No newline at end of file +startScripts.dependsOn shadowJar diff --git a/metadata-integration/java/openlineage-converter/build.gradle b/metadata-integration/java/openlineage-converter/build.gradle index d149104f089b36..1bf4a3c0fadb6a 100644 --- a/metadata-integration/java/openlineage-converter/build.gradle +++ b/metadata-integration/java/openlineage-converter/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'java-library' apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'maven-publish' -apply plugin: 'jacoco' +apply from: '../../../gradle/coverage/java-coverage.gradle' apply from: '../versioning.gradle' repositories { @@ -27,20 +27,16 @@ dependencies { testImplementation externalDependency.testng } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - test { forkEvery = 1 useJUnit() - finalizedBy jacocoTestReport } test { useJUnit() - finalizedBy jacocoTestReport + useTestNG() } + shadowJar { zip64 = true archiveClassifier = '' diff --git a/metadata-integration/java/spark-lineage-legacy/build.gradle b/metadata-integration/java/spark-lineage-legacy/build.gradle index 8db8a09f8cc813..d33290c16b3a56 100644 --- a/metadata-integration/java/spark-lineage-legacy/build.gradle +++ b/metadata-integration/java/spark-lineage-legacy/build.gradle @@ -6,7 +6,6 @@ apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' -apply plugin: 'jacoco' apply from: '../versioning.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -123,15 +122,6 @@ shadowJar { finalizedBy checkShadowJar } -checkShadowJar { - dependsOn shadowJar -} - - -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - test { forkEvery = 1 useJUnit() @@ -154,7 +144,6 @@ test { ] jvmArgs = sparkJava17CompatibleJvmArgs - finalizedBy jacocoTestReport } assemble { diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 41294fab7b24a9..516a77d59d50bd 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -4,6 +4,8 @@ plugins { id 'io.ebean' version "${ebeanVersion}" // Use the latest version from global build.gradle } +apply from: '../gradle/coverage/java-coverage.gradle' + configurations { enhance } diff --git a/metadata-io/metadata-io-api/build.gradle b/metadata-io/metadata-io-api/build.gradle index 5273177b752819..a49da206a69a8d 100644 --- a/metadata-io/metadata-io-api/build.gradle +++ b/metadata-io/metadata-io-api/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':entity-registry') implementation project(':metadata-service:services') diff --git a/metadata-jobs/common/build.gradle b/metadata-jobs/common/build.gradle index b0a3a6827b7297..120806431b1df5 100644 --- a/metadata-jobs/common/build.gradle +++ b/metadata-jobs/common/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation(project(':metadata-service:factories')) { exclude group: 'org.neo4j.test' diff --git a/metadata-jobs/mae-consumer-job/build.gradle b/metadata-jobs/mae-consumer-job/build.gradle index 4980f81d61bb4d..1fb46582c03817 100644 --- a/metadata-jobs/mae-consumer-job/build.gradle +++ b/metadata-jobs/mae-consumer-job/build.gradle @@ -4,6 +4,7 @@ plugins { id 'com.palantir.docker' } +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" ext { diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle index 6afc12c30e5f1f..b4990e289b10df 100644 --- a/metadata-jobs/mae-consumer/build.gradle +++ b/metadata-jobs/mae-consumer/build.gradle @@ -1,7 +1,7 @@ plugins { id 'java-library' } - +apply from: '../../gradle/coverage/java-coverage.gradle' apply plugin: 'pegasus' configurations { diff --git a/metadata-jobs/mce-consumer-job/build.gradle b/metadata-jobs/mce-consumer-job/build.gradle index 49c69626c211ca..d9b8965d7c8ac0 100644 --- a/metadata-jobs/mce-consumer-job/build.gradle +++ b/metadata-jobs/mce-consumer-job/build.gradle @@ -4,6 +4,7 @@ plugins { id 'com.palantir.docker' } +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" ext { diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle index 5ea24059a3ee33..2da3957c4bb218 100644 --- a/metadata-jobs/mce-consumer/build.gradle +++ b/metadata-jobs/mce-consumer/build.gradle @@ -3,6 +3,7 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' configurations { avro diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle index 35fdf0231c4064..ad17a443770857 100644 --- a/metadata-jobs/pe-consumer/build.gradle +++ b/metadata-jobs/pe-consumer/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' + configurations { avro } diff --git a/metadata-models-custom/build.gradle b/metadata-models-custom/build.gradle index 412c19194c7330..9babf9467e5d63 100644 --- a/metadata-models-custom/build.gradle +++ b/metadata-models-custom/build.gradle @@ -19,6 +19,8 @@ plugins { id 'pegasus' } +apply from: '../gradle/coverage/java-coverage.gradle' + if (project.hasProperty('projVersion')) { project.version = project.projVersion } else { diff --git a/metadata-models-validator/build.gradle b/metadata-models-validator/build.gradle index 1dae53e817ae14..c828ab041c27b3 100644 --- a/metadata-models-validator/build.gradle +++ b/metadata-models-validator/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(":entity-registry") implementation spec.product.pegasus.data diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index 179e1eac177ac8..e9379163ecaecc 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -6,6 +6,7 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../gradle/coverage/java-coverage.gradle' dependencies { api spec.product.pegasus.data diff --git a/metadata-operation-context/build.gradle b/metadata-operation-context/build.gradle index 650082ef0d25e6..71b61528d187bd 100644 --- a/metadata-operation-context/build.gradle +++ b/metadata-operation-context/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-library' } +apply from: '../gradle/coverage/java-coverage.gradle' dependencies { api project(':metadata-utils') diff --git a/metadata-service/auth-config/build.gradle b/metadata-service/auth-config/build.gradle index 8302e3b0c2fe67..de787e9f350dce 100644 --- a/metadata-service/auth-config/build.gradle +++ b/metadata-service/auth-config/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(path: ':metadata-models') implementation project(path: ':metadata-auth:auth-api') diff --git a/metadata-service/auth-filter/build.gradle b/metadata-service/auth-filter/build.gradle index 9d763ca11421b5..5e73246317677a 100644 --- a/metadata-service/auth-filter/build.gradle +++ b/metadata-service/auth-filter/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') implementation project(':metadata-service:auth-impl') diff --git a/metadata-service/auth-impl/build.gradle b/metadata-service/auth-impl/build.gradle index 4f4b0658caf249..b5c20ccabe6d2d 100644 --- a/metadata-service/auth-impl/build.gradle +++ b/metadata-service/auth-impl/build.gradle @@ -2,6 +2,7 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' compileJava { options.debug = true diff --git a/metadata-service/auth-servlet-impl/build.gradle b/metadata-service/auth-servlet-impl/build.gradle index 29e452472358b8..21fe63778cbeb7 100644 --- a/metadata-service/auth-servlet-impl/build.gradle +++ b/metadata-service/auth-servlet-impl/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') implementation project(':metadata-service:auth-impl') diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle index f912e2ac01f0b6..3c7e38b004fae7 100644 --- a/metadata-service/configuration/build.gradle +++ b/metadata-service/configuration/build.gradle @@ -1,6 +1,8 @@ plugins { id 'java-library' } + +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" dependencies { diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index 5e52efd245b7fa..501c46d64d6f9c 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' dependencies { api project(':metadata-io') diff --git a/metadata-service/graphql-servlet-impl/build.gradle b/metadata-service/graphql-servlet-impl/build.gradle index 57676982421186..6848696db6788a 100644 --- a/metadata-service/graphql-servlet-impl/build.gradle +++ b/metadata-service/graphql-servlet-impl/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':datahub-graphql-core') implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/openapi-analytics-servlet/build.gradle b/metadata-service/openapi-analytics-servlet/build.gradle index 5d1372a293775b..45e0ee1758ebc1 100644 --- a/metadata-service/openapi-analytics-servlet/build.gradle +++ b/metadata-service/openapi-analytics-servlet/build.gradle @@ -3,6 +3,8 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/openapi-entity-servlet/build.gradle b/metadata-service/openapi-entity-servlet/build.gradle index 4c2d587a81fd78..a47e487f871c6f 100644 --- a/metadata-service/openapi-entity-servlet/build.gradle +++ b/metadata-service/openapi-entity-servlet/build.gradle @@ -3,6 +3,8 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle index e26b1ceea1a3c6..77679790f25dea 100644 --- a/metadata-service/openapi-servlet/build.gradle +++ b/metadata-service/openapi-servlet/build.gradle @@ -3,6 +3,8 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/plugin/build.gradle b/metadata-service/plugin/build.gradle index 501589c24d60a2..526744ceb85c59 100644 --- a/metadata-service/plugin/build.gradle +++ b/metadata-service/plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation 'org.apache.commons:commons-lang3:3.12.0' diff --git a/metadata-service/restli-api/build.gradle b/metadata-service/restli-api/build.gradle index 505320e8267eed..38a3e9298ff963 100644 --- a/metadata-service/restli-api/build.gradle +++ b/metadata-service/restli-api/build.gradle @@ -2,6 +2,8 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { dataModel project(':metadata-models') dataModel spec.product.pegasus.restliCommon diff --git a/metadata-service/restli-client-api/build.gradle b/metadata-service/restli-client-api/build.gradle index c2dfd4d2a1344c..4e488a98023cb4 100644 --- a/metadata-service/restli-client-api/build.gradle +++ b/metadata-service/restli-client-api/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { api project(path: ':metadata-service:restli-api', configuration: 'restClient') api project(':metadata-operation-context') diff --git a/metadata-service/restli-client/build.gradle b/metadata-service/restli-client/build.gradle index 587ff5b2ef0531..92b34330166335 100644 --- a/metadata-service/restli-client/build.gradle +++ b/metadata-service/restli-client/build.gradle @@ -3,6 +3,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { api project(':metadata-service:restli-api') api project(':metadata-auth:auth-api') diff --git a/metadata-service/restli-servlet-impl/build.gradle b/metadata-service/restli-servlet-impl/build.gradle index 6b68abfe7fb15c..dc426bf5639b17 100644 --- a/metadata-service/restli-servlet-impl/build.gradle +++ b/metadata-service/restli-servlet-impl/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' + sourceSets { integTest { compileClasspath += sourceSets.main.output diff --git a/metadata-service/schema-registry-api/build.gradle b/metadata-service/schema-registry-api/build.gradle index 201e98beb490e0..525eceda685be6 100644 --- a/metadata-service/schema-registry-api/build.gradle +++ b/metadata-service/schema-registry-api/build.gradle @@ -3,6 +3,7 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' dependencies { // Dependencies for open api diff --git a/metadata-service/schema-registry-servlet/build.gradle b/metadata-service/schema-registry-servlet/build.gradle index f19237c32cee78..94ca8fe03822e3 100644 --- a/metadata-service/schema-registry-servlet/build.gradle +++ b/metadata-service/schema-registry-servlet/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-service:factories') implementation project(':metadata-service:schema-registry-api') diff --git a/metadata-service/services/build.gradle b/metadata-service/services/build.gradle index ea1ff32cb3838b..0e84580cc04da5 100644 --- a/metadata-service/services/build.gradle +++ b/metadata-service/services/build.gradle @@ -3,6 +3,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + configurations { enhance } diff --git a/metadata-service/servlet/build.gradle b/metadata-service/servlet/build.gradle index f961bf6a9de7eb..725352e69d52b2 100644 --- a/metadata-service/servlet/build.gradle +++ b/metadata-service/servlet/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-io') implementation project(':datahub-graphql-core') diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index ab9019e470dbca..5e00207c1726ee 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -3,6 +3,7 @@ plugins { id 'com.palantir.docker' } +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" ext { diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle index 919d93c5f9fe19..07ce50993655d2 100644 --- a/metadata-utils/build.gradle +++ b/metadata-utils/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: '../gradle/coverage/java-coverage.gradle' + dependencies { api externalDependency.avro implementation externalDependency.commonsLang diff --git a/mock-entity-registry/build.gradle b/mock-entity-registry/build.gradle index 8242d6451dd606..3a6ce0eedcba6e 100644 --- a/mock-entity-registry/build.gradle +++ b/mock-entity-registry/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':entity-registry') } \ No newline at end of file From e19af9ec356d1e61cb41516355bcbdc16aede14b Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 3 Dec 2024 00:45:53 -0600 Subject: [PATCH 113/174] chore(docs): Update v_0_3_7.md - v0.3.7.5 (#12005) --- docs/managed-datahub/release-notes/v_0_3_7.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index 94cbdd79dbf5ef..be3a2d97514efa 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -19,6 +19,13 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies ## Release Changelog --- +### v0.3.7.5 + +- [GMS] Fix upstream lineage patching when path contained encoded slash +- [UI] Fix merging siblings schema with v1 and v2 fields +- [UI] Fix display nullable in schema field drawer +- [Ingestion] Reduce Data Product write volume from unset side-effect + ### v0.3.7.4 - [#11935](https://github.com/datahub-project/datahub/pull/11935) - Added environment variable for enabling stricter URN validation rules `STRICT_URN_VALIDATION_ENABLED` [[1](https://datahubproject.io/docs/what/urn/#restrictions)]. From 7429075ebd93c7138ca341f4c217343f90c03d19 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 2 Dec 2024 23:23:22 -0800 Subject: [PATCH 114/174] =?UTF-8?q?feat(java-sdk):=20add=20utils=20classes?= =?UTF-8?q?=20to=20give=20equivalence=20with=20python=20uti=E2=80=A6=20(#1?= =?UTF-8?q?2002)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/datahub-client/build.gradle | 27 +++- .../scripts/container_key_guid_generator.py | 122 ++++++++++++++++++ .../models/util/ContainerKey.java | 42 ++++++ .../models/util/DataHubGuidGenerator.java | 41 ++++++ .../models/util/DataHubKey.java | 32 +++++ .../models/util/DatabaseKey.java | 30 +++++ .../datahubproject/models/util/FieldPath.java | 88 +++++++++++++ .../datahubproject/models/util/SchemaKey.java | 30 +++++ .../models/util/DataHubGuidGeneratorTest.java | 61 +++++++++ .../models/util/DatabaseKeyTest.java | 51 ++++++++ .../models/util/FieldPathTest.java | 83 ++++++++++++ .../models/util/SchemaKeyTest.java | 62 +++++++++ .../models/util/TestHelper.java | 114 ++++++++++++++++ 13 files changed, 782 insertions(+), 1 deletion(-) create mode 100644 metadata-integration/java/datahub-client/scripts/container_key_guid_generator.py create mode 100644 metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/ContainerKey.java create mode 100644 metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubGuidGenerator.java create mode 100644 metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubKey.java create mode 100644 metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DatabaseKey.java create mode 100644 metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/FieldPath.java create mode 100644 metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/SchemaKey.java create mode 100644 metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DataHubGuidGeneratorTest.java create mode 100644 metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DatabaseKeyTest.java create mode 100644 metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/FieldPathTest.java create mode 100644 metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/SchemaKeyTest.java create mode 100644 metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/TestHelper.java diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 87a9623bbf062f..af71227809d2a7 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -20,6 +20,7 @@ dependencies { api project(':entity-registry') api project(':metadata-integration:java:datahub-event') implementation project(':metadata-integration:java:datahub-schematron:lib') + implementation(externalDependency.kafkaAvroSerializer) { exclude group: "org.apache.avro" } @@ -60,10 +61,35 @@ task copyAvroSchemas { compileJava.dependsOn copyAvroSchemas +// Add Python environment validation task +task validatePythonEnv { + doFirst { + def venvPath = System.getProperty('python.venv.path', '../../../metadata-ingestion/venv') + def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') + def pythonExe = isWindows ? "${venvPath}/Scripts/python.exe" : "${venvPath}/bin/python" + + def result = exec { + commandLine pythonExe, "-c", "import sys; print(sys.executable)" + ignoreExitValue = true + standardOutput = new ByteArrayOutputStream() + errorOutput = new ByteArrayOutputStream() + } + + if (result.exitValue != 0) { + throw new GradleException("Python virtual environment not properly set up at ${venvPath}") + } + } +} + test { // to avoid simultaneous executions of tests when complete build is run mustRunAfter(":metadata-io:test") useJUnit() + // Add Python environment configuration + dependsOn validatePythonEnv + dependsOn tasks.getByPath(":metadata-ingestion:installDev") + systemProperty 'python.venv.path', System.getProperty('python.venv.path', '../../../metadata-ingestion/venv') + finalizedBy jacocoTestReport } task checkShadowJar(type: Exec) { @@ -111,7 +137,6 @@ shadowJar { relocate 'org.checkerframework', 'datahub.shaded.org.checkerframework' relocate 'com.google.errorprone', 'datahub.shaded.com.google.errorprone' // Below jars added for kafka emitter only -// relocate 'org.apache.avro', 'datahub.shaded.org.apache.avro' relocate 'com.thoughtworks.paranamer', 'datahub.shaded.com.thoughtworks.paranamer' relocate 'org.xerial.snappy', 'datahub.shaded.org.xerial.snappy' relocate 'org.apache.kafka', 'datahub.shaded.org.apache.kafka' diff --git a/metadata-integration/java/datahub-client/scripts/container_key_guid_generator.py b/metadata-integration/java/datahub-client/scripts/container_key_guid_generator.py new file mode 100644 index 00000000000000..9fc18a85426bd8 --- /dev/null +++ b/metadata-integration/java/datahub-client/scripts/container_key_guid_generator.py @@ -0,0 +1,122 @@ +import click +from typing import Dict, Any +import json +from dataclasses import dataclass +from abc import ABC, abstractmethod +from datahub.emitter.mcp_builder import DatabaseKey, SchemaKey + + +class URNGenerator(ABC): + @abstractmethod + def generate(self, args: Dict[str, Any]) -> str: + pass + + +class DatabaseURNGenerator(URNGenerator): + def generate(self, args: Dict[str, Any]) -> str: + required_fields = ["platform", "database"] + for field in required_fields: + if field not in args: + raise ValueError(f"Missing required field: {field}") + + all_fields = required_fields + ["instance"] + for arg in args: + if arg not in all_fields: + raise ValueError(f"Invalid field: {arg}") + + database_key = DatabaseKey( + platform=args["platform"], + instance=args.get("instance"), + database=args["database"], + ) + return database_key.as_urn() + + +class SchemaURNGenerator(URNGenerator): + def generate(self, args: Dict[str, Any]) -> str: + required_fields = ["platform", "database", "schema"] + all_fields = required_fields + ["instance", "env"] + for field in required_fields: + if field not in args: + raise ValueError(f"Missing required field: {field}") + + for arg in args: + if arg not in all_fields: + raise ValueError(f"Invalid field: {arg}") + + schema_key = SchemaKey( + platform=args["platform"], + instance=args.get("instance"), + env=args.get("env"), + database=args["database"], + schema=args["schema"], + ) + return schema_key.as_urn() + + +URN_GENERATORS = { + "database": DatabaseURNGenerator(), + "schema": SchemaURNGenerator(), +} + + +def validate_key_value(ctx, param, value): + if not value: + return {} + + result = {} + for item in value: + try: + key, val = item.split("=", 1) + result[key.strip()] = val.strip() + except ValueError: + raise click.BadParameter( + f"Invalid key-value pair: {item}. Format should be key=value" + ) + return result + + +@click.command() +@click.option( + "--container-type", + type=click.Choice(["database", "schema"]), + required=True, + help="The type of container to generate a URN for", +) +@click.option( + "--param", + "-p", + multiple=True, + callback=validate_key_value, + help="Parameters in key=value format. Can be used multiple times.", +) +@click.option( + "--output-format", + type=click.Choice(["text", "json"]), + default="text", + help="Output format for the URN", +) +def generate_urn(container_type: str, param: Dict[str, str], output_format: str): + """Generate URNs for different types of containers. + + Example usage: + ./container_urn_generator.py --container-type database -p platform=test-platform -p instance=DEV -p database=test-database + """ + try: + generator = URN_GENERATORS[container_type] + urn = generator.generate(param) + + if output_format == "json": + result = {"urn": urn, "container_type": container_type, "parameters": param} + click.echo(json.dumps(result, indent=2)) + else: + click.echo(urn) + + except KeyError as e: + raise click.UsageError(f"Unknown container type: {container_type}") + except ValueError as e: + raise click.UsageError(str(e)) + + +if __name__ == "__main__": + generate_urn() diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/ContainerKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/ContainerKey.java new file mode 100644 index 00000000000000..5bc6c829dcaa1c --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/ContainerKey.java @@ -0,0 +1,42 @@ +package io.datahubproject.models.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.linkedin.common.urn.Urn; +import java.util.HashMap; +import java.util.Map; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@EqualsAndHashCode(callSuper = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public abstract class ContainerKey extends DataHubKey { + private String platform; + private String instance; + + private static final String URN_PREFIX = "urn:li:container:"; + private static final String URN_ENTITY = "container"; + private static final String PLATFORM_MAP_FIELD = "platform"; + private static final String INSTANCE_MAP_FIELD = "instance"; + + @Override + public Map guidDict() { + + Map bag = new HashMap<>(); + if (platform != null) bag.put(PLATFORM_MAP_FIELD, platform); + if (instance != null) bag.put(INSTANCE_MAP_FIELD, instance); + + return bag; + } + + public String asUrnString() { + String guid = guid(); + return URN_PREFIX + guid; + } + + public Urn asUrn() { + return Urn.createFromTuple(URN_ENTITY, guid()); + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubGuidGenerator.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubGuidGenerator.java new file mode 100644 index 00000000000000..ea67b7fab17817 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubGuidGenerator.java @@ -0,0 +1,41 @@ +package io.datahubproject.models.util; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.security.MessageDigest; +import java.util.Map; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DataHubGuidGenerator { + private static final ObjectMapper objectMapper = new ObjectMapper(); + + @SneakyThrows + public static String dataHubGuid(Map obj) { + // Configure ObjectMapper for consistent serialization + objectMapper.configure( + com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + + // Convert map to JSON string with sorted keys + String jsonKey = objectMapper.writeValueAsString(obj); + + // Generate MD5 hash + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] hashBytes = md.digest(jsonKey.getBytes()); + + // Convert byte array to hexadecimal string + StringBuilder hexString = new StringBuilder(); + for (byte hashByte : hashBytes) { + String hex = Integer.toHexString(0xff & hashByte); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + + if (log.isDebugEnabled()) { + log.debug("DataHub Guid for {} is : {}", jsonKey, hexString); + } + return hexString.toString(); + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubKey.java new file mode 100644 index 00000000000000..a047dd88409359 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DataHubKey.java @@ -0,0 +1,32 @@ +package io.datahubproject.models.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Map; +import lombok.Data; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@JsonInclude(JsonInclude.Include.NON_NULL) +public abstract class DataHubKey { + // Static ObjectMapper instance since it's thread-safe and expensive to create + protected static final ObjectMapper MAPPER = new ObjectMapper(); + // Static TypeReference instance since it doesn't change + private static final TypeReference> MAP_TYPE_REFERENCE = + new TypeReference>() {}; + + static { + MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); + } + + public Map guidDict() { + return MAPPER.convertValue(this, MAP_TYPE_REFERENCE); + } + + public String guid() { + Map bag = guidDict(); + return DataHubGuidGenerator.dataHubGuid(bag); + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DatabaseKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DatabaseKey.java new file mode 100644 index 00000000000000..87a79ea2e74404 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/DatabaseKey.java @@ -0,0 +1,30 @@ +package io.datahubproject.models.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.Map; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@EqualsAndHashCode(callSuper = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DatabaseKey extends ContainerKey { + private String database; + + private static final String DATABASE_MAP_FIELD = "database"; + + @Override + public Map guidDict() { + // Get the parent's GUID dictionary first + Map bag = super.guidDict(); + + // Add the database field if it's not null + if (database != null) { + bag.put(DATABASE_MAP_FIELD, database); + } + + return bag; + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/FieldPath.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/FieldPath.java new file mode 100644 index 00000000000000..ab8c11ae9b67d6 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/FieldPath.java @@ -0,0 +1,88 @@ +package io.datahubproject.models.util; + +import lombok.NonNull; + +public class FieldPath { + private final String rawFieldPath; + private final String[] segments; + private final String version; + private final String simplePath; + + public FieldPath(@NonNull String rawFieldPath) { + this.rawFieldPath = rawFieldPath; + this.segments = rawFieldPath.split("\\."); + this.version = computeVersion(); + this.simplePath = computeSimplePath(); + } + + public boolean isTopLevel() { + return depth() == 1; + } + + /** + * Returns the logical depth of the field path, ignoring type and version metadata Example: + * "[version=2.0][type=Foo].address.[type=union].[type=string].street" -> depth = 2 + * + * @return The logical depth of the field path + */ + public int depth() { + String[] segments = simplePath().split("\\."); + return segments.length; + } + + public String leafFieldName() { + return segments[segments.length - 1]; + } + + /** + * Extracts the version number from the field path. Example: rawFieldPath = "a.b.c" -> version() = + * "1" rawFieldPath = "[version=2].a.b.c" -> version() = "2" rawFieldPath = "[version=20].a.b.c" + * -> version() = "20" + * + * @return String representing the version number + */ + private String computeVersion() { + if (rawFieldPath == null) { + return "1"; + } + + // Check for version pattern [version=X] where X can be any non-bracket characters + java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("\\[version=([^\\]]+)\\]"); + java.util.regex.Matcher matcher = pattern.matcher(rawFieldPath); + + if (matcher.find()) { + return matcher.group(1); + } + + return "1"; + } + + public String version() { + return version; + } + + /** + * Returns the simplified path without version, type, or other metadata Example: + * "[version=2.0][type=Foo].address.[type=union].[type=string].street" -> "address.street" + * + * @return The simplified field path + */ + private String computeSimplePath() { + if (rawFieldPath == null) { + return ""; + } + // Remove all metadata blocks [xxx=yyy] + String simplified = rawFieldPath.replaceAll("\\[.*?\\]", ""); + // Replace all "double dots" with a single dot + simplified = simplified.replaceAll("\\.+", "."); + // Replace all leading and trailing dots + simplified = simplified.replaceAll("^\\.|\\.$", ""); + // Remove any trailing metadata blocks without dots + simplified = simplified.replaceAll("\\[.*?\\]", ""); + return simplified; + } + + public String simplePath() { + return simplePath; + } +} diff --git a/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/SchemaKey.java b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/SchemaKey.java new file mode 100644 index 00000000000000..800ee436147492 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/main/java/io/datahubproject/models/util/SchemaKey.java @@ -0,0 +1,30 @@ +package io.datahubproject.models.util; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.Map; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +@EqualsAndHashCode(callSuper = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SchemaKey extends DatabaseKey { + private String schema; + + private static final String SCHEMA_MAP_FIELD = "schema"; + + @Override + public Map guidDict() { + // Get the parent's GUID dictionary first + Map bag = super.guidDict(); + + // Add the database field if it's not null + if (schema != null) { + bag.put(SCHEMA_MAP_FIELD, schema); + } + + return bag; + } +} diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DataHubGuidGeneratorTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DataHubGuidGeneratorTest.java new file mode 100644 index 00000000000000..8ee65d51de92d2 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DataHubGuidGeneratorTest.java @@ -0,0 +1,61 @@ +package io.datahubproject.models.util; + +import static org.junit.Assert.*; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Map; +import org.junit.Test; + +public class DataHubGuidGeneratorTest { + + @Test + public void testGuidGeneration() throws NoSuchAlgorithmException, JsonProcessingException { + // Test data + Map obj = new HashMap<>(); + obj.put("container", "test-container"); + + // Generate GUID + String guid = DataHubGuidGenerator.dataHubGuid(obj); + + // Assert + assertEquals("4d90f727b9d10ba7cea297dc8b427985", guid); + } + + @Test + public void testContainerUrnGeneration() { + // Test data + DatabaseKey databaseKey = + DatabaseKey.builder() + .platform("test-platform") + .instance("DEV") + .database("test-database") + .build(); + + System.out.println(databaseKey.guidDict()); + + // Generate URN + String urn = databaseKey.asUrnString(); + // With instance + String expectedUrn = "urn:li:container:e40f103ea7c6def4f4b24cd858d5e412"; + + // Assert + assertEquals(expectedUrn, urn); + } + + @Test + public void testContainerUrnGenerationNoInstance() { + // Test data + ContainerKey containerKey = + DatabaseKey.builder().platform("test-platform").database("test-database").build(); + + // Generate URN + String urn = containerKey.asUrnString(); + // Without instance + String expectedUrn = "urn:li:container:1929d86c0a92e2d3bb9ba193c8c2b66f"; + + // Assert + assertEquals(expectedUrn, urn); + } +} diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DatabaseKeyTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DatabaseKeyTest.java new file mode 100644 index 00000000000000..08c6e9ce33a68e --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/DatabaseKeyTest.java @@ -0,0 +1,51 @@ +package io.datahubproject.models.util; + +import static org.junit.Assert.*; + +import java.util.Map; +import org.junit.Test; + +public class DatabaseKeyTest { + @Test + public void testDatabaseUrnGeneration() { + // Test data + DatabaseKey databaseKey = + DatabaseKey.builder() + .platform("test-platform") + .instance("DEV") + .database("test-database") + .build(); + + System.out.println(databaseKey.guidDict()); + + // Generate URN + String urn = databaseKey.asUrnString(); + // With instance + // "urn:li:container:e40f103ea7c6def4f4b24cd858d5e412"; + String expectedUrn = + TestHelper.generateContainerKeyGuid( + "database", + Map.of("platform", "test-platform", "instance", "DEV", "database", "test-database")); + + // Assert + assertEquals(expectedUrn, urn); + } + + @Test + public void testDatabaseUrnGenerationNoInstance() { + // Test data + ContainerKey containerKey = + DatabaseKey.builder().platform("test-platform").database("test-database").build(); + + // Generate URN + String urn = containerKey.asUrnString(); + // Without instance + // "urn:li:container:1929d86c0a92e2d3bb9ba193c8c2b66f"; + String expectedUrn = + TestHelper.generateContainerKeyGuid( + "database", Map.of("platform", "test-platform", "database", "test-database")); + + // Assert + assertEquals(expectedUrn, urn); + } +} diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/FieldPathTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/FieldPathTest.java new file mode 100644 index 00000000000000..55b97f735393ca --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/FieldPathTest.java @@ -0,0 +1,83 @@ +package io.datahubproject.models.util; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class FieldPathTest { + + @Test + public void testSimplePathConstruction() { + FieldPath path = new FieldPath("field1.field2.field3"); + assertEquals("field1.field2.field3", path.simplePath()); + assertEquals("1", path.version()); + assertEquals("field3", path.leafFieldName()); + assertEquals(3, path.depth()); + assertFalse(path.isTopLevel()); + } + + @Test + public void testTopLevelPath() { + FieldPath path = new FieldPath("singleField"); + assertTrue(path.isTopLevel()); + assertEquals(1, path.depth()); + assertEquals("singleField", path.leafFieldName()); + assertEquals("singleField", path.simplePath()); + } + + @Test + public void testVersionExtraction() { + FieldPath path = new FieldPath("[version=2.0].field1.field2"); + assertEquals("2.0", path.version()); + assertEquals("field1.field2", path.simplePath()); + } + + @Test + public void testComplexPathWithMetadata() { + FieldPath path = + new FieldPath("[version=2.0][type=Foo].address.[type=union].[type=string].street"); + assertEquals("2.0", path.version()); + assertEquals("address.street", path.simplePath()); + assertEquals("street", path.leafFieldName()); + assertEquals(2, path.depth()); + } + + @Test + public void testPathWithMultipleMetadataBlocks() { + FieldPath path = + new FieldPath("[version=3][type=Record][nullable=true].user.details[type=union].name"); + System.out.println(path.simplePath()); + assertEquals("3", path.version()); + assertEquals("user.details.name", path.simplePath()); + assertEquals("name", path.leafFieldName()); + assertEquals(3, path.depth()); + } + + @Test + public void testPathWithTrailingMetadata() { + FieldPath path = new FieldPath("field1.field2[type=string]"); + assertEquals("field1.field2", path.simplePath()); + assertEquals("1", path.version()); + } + + @Test + public void testPathWithOnlyMetadata() { + FieldPath path = new FieldPath("[type=string][version=4]"); + assertEquals("", path.simplePath()); + assertEquals("4", path.version()); + } + + @Test + public void testDepthCalculationWithComplexPath() { + FieldPath path = + new FieldPath("[version=2].user.[type=record].address.[type=union].[type=string].street"); + assertEquals(3, path.depth()); + assertEquals("user.address.street", path.simplePath()); + } + + @Test + public void testLeafFieldNameWithMetadata() { + FieldPath path = new FieldPath("field1.field2[type=string]"); + assertEquals("field2[type=string]", path.leafFieldName()); + } +} diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/SchemaKeyTest.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/SchemaKeyTest.java new file mode 100644 index 00000000000000..7d40491a9b1d86 --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/SchemaKeyTest.java @@ -0,0 +1,62 @@ +package io.datahubproject.models.util; + +import static org.junit.Assert.*; + +import java.util.Map; +import org.junit.Test; + +public class SchemaKeyTest { + @Test + public void testSchemaUrnGeneration() { + // Test data + SchemaKey schemaKey = + SchemaKey.builder() + .platform("test-platform") + .instance("DEV") + .database("test-database") + .schema("test-schema") + .build(); + + System.out.println(schemaKey.guidDict()); + + // Generate URN + String urn = schemaKey.asUrnString(); + // With instance + String expectedUrn = + TestHelper.generateContainerKeyGuid( + "schema", + Map.of( + "platform", "test-platform", + "instance", "DEV", + "database", "test-database", + "schema", "test-schema")); + + // Assert + assertEquals(expectedUrn, urn); + } + + @Test + public void testSchemaUrnGenerationNoInstance() { + // Test data + ContainerKey containerKey = + SchemaKey.builder() + .platform("test-platform") + .database("test-database") + .schema("test-schema") + .build(); + + // Generate URN + String urn = containerKey.asUrnString(); + // Without instance + String expectedUrn = + TestHelper.generateContainerKeyGuid( + "schema", + Map.of( + "platform", "test-platform", + "database", "test-database", + "schema", "test-schema")); + + // Assert + assertEquals(expectedUrn, urn); + } +} diff --git a/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/TestHelper.java b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/TestHelper.java new file mode 100644 index 00000000000000..e150ceda16262b --- /dev/null +++ b/metadata-integration/java/datahub-client/src/test/java/io/datahubproject/models/util/TestHelper.java @@ -0,0 +1,114 @@ +package io.datahubproject.models.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +public class TestHelper { + + private static final String TEST_RESOURCES_DIR = "src/test/resources"; + // private static final String TEMP_OUTPUT_DIR = "build/test-outputs"; + private static final String PYTHON_SCRIPT = "scripts/container_key_guid_generator.py"; + private static final String VENV_PATH = + "../../../metadata-ingestion/venv"; // Adjust this path to your venv location + + public static void setup() { + // Create output directory if it doesn't exist + // new File(TEMP_OUTPUT_DIR).mkdirs(); + + // Verify venv exists + if (!new File(VENV_PATH).exists()) { + throw new RuntimeException("Virtual environment not found at " + VENV_PATH); + } + } + + private static ProcessBuilder createPythonProcessBuilder(String... args) { + ProcessBuilder pb; + String os = System.getProperty("os.name").toLowerCase(); + + if (os.contains("windows")) { + // Windows paths + String pythonPath = Paths.get(VENV_PATH, "Scripts", "python").toString(); + pb = + new ProcessBuilder( + Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new)); + } else { + // Unix-like paths + String pythonPath = Paths.get(VENV_PATH, "bin", "python").toString(); + pb = + new ProcessBuilder( + Stream.concat(Stream.of(pythonPath), Stream.of(args)).toArray(String[]::new)); + } + + // Add virtual environment to PYTHONPATH + Map env = pb.environment(); + String sitePkgPath = + Paths.get( + VENV_PATH, + os.contains("windows") ? "Lib/site-packages" : "lib/python3.x/site-packages") + .toString(); + + String pythonPath = env.getOrDefault("PYTHONPATH", ""); + env.put("PYTHONPATH", pythonPath + File.pathSeparator + sitePkgPath); + + return pb; + } + + /** + * Executes the container key GUID generator Python script with the given parameters + * + * @param containerType The type of container (e.g., "database") + * @param parameters Map of parameter key-value pairs + * @return The generated URN as a string + * @throws RuntimeException if the Python script execution fails + */ + public static String generateContainerKeyGuid( + String containerType, Map parameters) { + List command = new ArrayList<>(); + command.add(PYTHON_SCRIPT); + command.add("--container-type"); + command.add(containerType); + + // Add each parameter as -p key=value + for (Map.Entry entry : parameters.entrySet()) { + command.add("-p"); + command.add(entry.getKey() + "=" + entry.getValue()); + } + + try { + System.out.println("Executing Python script: " + String.join(" ", command)); + String[] commandArray = command.toArray(new String[0]); + + Process process = createPythonProcessBuilder(commandArray).start(); + + // Read the output + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = reader.readLine(); + + // Wait for the process to complete + int exitCode = process.waitFor(); + + if (exitCode != 0) { + // Read error stream if the script failed + BufferedReader errorReader = + new BufferedReader(new InputStreamReader(process.getErrorStream())); + StringBuilder errorMessage = new StringBuilder(); + String errorLine; + while ((errorLine = errorReader.readLine()) != null) { + errorMessage.append(errorLine).append("\n"); + } + throw new RuntimeException( + "Python script failed with exit code " + exitCode + ": " + errorMessage); + } + + return line != null ? line.trim() : ""; + } catch (Exception e) { + throw new RuntimeException("Failed to execute Python script", e); + } + } +} From 16a02411c343b2527fcac3044cf863a9ab9ae843 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Tue, 3 Dec 2024 10:48:04 +0100 Subject: [PATCH 115/174] fix(ingest/sagemaker): Gracefully handle missing model group (#12000) --- .../sagemaker_processors/feature_groups.py | 13 +++++--- .../source/aws/sagemaker_processors/models.py | 31 ++++++++++++++++++- .../unit/sagemaker/test_sagemaker_source.py | 15 +++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py index b8b96c6306a3bb..c4561b9d9e676a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/feature_groups.py @@ -1,3 +1,5 @@ +import logging +import textwrap from dataclasses import dataclass from typing import TYPE_CHECKING, Iterable, List @@ -28,6 +30,8 @@ FeatureGroupSummaryTypeDef, ) +logger = logging.getLogger(__name__) + @dataclass class FeatureGroupProcessor: @@ -197,11 +201,12 @@ def get_feature_wu( full_table_name = f"{glue_database}.{glue_table}" - self.report.report_warning( - full_table_name, - f"""Note: table {full_table_name} is an AWS Glue object. + logging.info( + textwrap.dedent( + f"""Note: table {full_table_name} is an AWS Glue object. This source does not ingest all metadata for Glue tables. To view full table metadata, run Glue ingestion - (see https://datahubproject.io/docs/metadata-ingestion/#aws-glue-glue)""", + (see https://datahubproject.io/docs/generated/ingestion/sources/glue)""" + ) ) feature_sources.append( diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py index eef2b26ee08f2e..0f433aaecf2d96 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/models.py @@ -1,3 +1,4 @@ +import logging from collections import defaultdict from dataclasses import dataclass, field from datetime import datetime @@ -65,6 +66,8 @@ "Unknown": DeploymentStatusClass.UNKNOWN, } +logger = logging.getLogger(__name__) + @dataclass class ModelProcessor: @@ -385,6 +388,26 @@ def strip_quotes(string: str) -> str: model_metrics, ) + @staticmethod + def get_group_name_from_arn(arn: str) -> str: + """ + Extract model package group name from a SageMaker ARN. + + Args: + arn (str): Full ARN of the model package group + + Returns: + str: Name of the model package group + + Example: + >>> ModelProcessor.get_group_name_from_arn("arn:aws:sagemaker:eu-west-1:123456789:model-package-group/my-model-group") + 'my-model-group' + """ + logger.debug( + f"Extracting group name from ARN: {arn} because group was not seen before" + ) + return arn.split("/")[-1] + def get_model_wu( self, model_details: "DescribeModelOutputTypeDef", @@ -425,8 +448,14 @@ def get_model_wu( model_group_arns = model_uri_groups | model_image_groups model_group_names = sorted( - [self.group_arn_to_name[x] for x in model_group_arns] + [ + self.group_arn_to_name[x] + if x in self.group_arn_to_name + else self.get_group_name_from_arn(x) + for x in model_group_arns + ] ) + model_group_urns = [ builder.make_ml_model_group_urn("sagemaker", x, self.env) for x in model_group_names diff --git a/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py index 2450e6fa8fe564..138319feb3db67 100644 --- a/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py +++ b/metadata-ingestion/tests/unit/sagemaker/test_sagemaker_source.py @@ -241,3 +241,18 @@ def test_sagemaker_ingest(tmp_path, pytestconfig): output_path=tmp_path / "sagemaker_mces.json", golden_path=test_resources_dir / "sagemaker_mces_golden.json", ) + + +def test_doc_test_run(): + import doctest + + import datahub.ingestion.source.aws.sagemaker_processors.models + + assert ( + doctest.testmod( + datahub.ingestion.source.aws.sagemaker_processors.models, + raise_on_error=True, + verbose=True, + ).attempted + == 1 + ) From 5d7298ea07fc74cd93b87f6f9e368d76295565bc Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 3 Dec 2024 18:32:24 +0530 Subject: [PATCH 116/174] fix(ingest/gc): typo fix, do not delete empty entities (#12011) --- .../ingestion/source/gc/dataprocess_cleanup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index ca67cd6daa045b..2d042c7ea68ec7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -114,11 +114,11 @@ class DataProcessCleanupConfig(ConfigModel): ) delete_empty_data_jobs: bool = Field( - True, description="Wether to delete Data Jobs without runs" + True, description="Whether to delete Data Jobs without runs" ) delete_empty_data_flows: bool = Field( - True, description="Wether to delete Data Flows without runs" + True, description="Whether to delete Data Flows without runs" ) hard_delete_entities: bool = Field( @@ -128,7 +128,7 @@ class DataProcessCleanupConfig(ConfigModel): batch_size: int = Field( 500, - description="The number of entities to get in a batch from GraphQL", + description="The number of entities to get in a batch from API", ) max_workers: int = Field( @@ -173,9 +173,9 @@ class DataProcessCleanup: """ This source is a maintenance source which cleans up old/unused aspects. - Currently it only supports:. + Currently it only supports: - DataFlow - -DataJob + - DataJob - DataProcessInstance """ @@ -267,7 +267,7 @@ def delete_entity(self, urn: str, type: str) -> None: if self.dry_run: logger.info( - f"Dry run is on otherwise it would have deleted {urn} with hard deletion is{self.config.hard_delete_entities}" + f"Dry run is on otherwise it would have deleted {urn} with hard deletion is {self.config.hard_delete_entities}" ) return From db7e51bf10b20a28d5c70366f32517688e7de167 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 3 Dec 2024 19:23:10 +0530 Subject: [PATCH 117/174] fix(ingest/gc): do not cleanup empty job/flow (#12013) --- .../src/datahub/ingestion/source/gc/dataprocess_cleanup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 2d042c7ea68ec7..90641b7059ca40 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -114,11 +114,11 @@ class DataProcessCleanupConfig(ConfigModel): ) delete_empty_data_jobs: bool = Field( - True, description="Whether to delete Data Jobs without runs" + False, description="Whether to delete Data Jobs without runs" ) delete_empty_data_flows: bool = Field( - True, description="Whether to delete Data Flows without runs" + False, description="Whether to delete Data Flows without runs" ) hard_delete_entities: bool = Field( From aca1cd7fe41627e31b1e5ce9df3f1b1474b5c35b Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:25:24 -0600 Subject: [PATCH 118/174] fix(test): fix metadata-io tests (#12006) --- .../search/fixtures/SampleDataFixtureTestBase.java | 8 ++++---- .../src/test/resources/search_config_fixture_test.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java index 504eb5f5fc13db..fd663de40e0050 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/SampleDataFixtureTestBase.java @@ -1367,8 +1367,8 @@ public void testScrollAcrossEntities() throws IOException { resultUrns.addAll(result.getEntities().stream().map(SearchEntity::getEntity).toList()); scrollId = result.getScrollId(); } while (scrollId != null); - // expect 2 total matching results - assertEquals(totalResults, 2, String.format("query `%s` Results: %s", query, resultUrns)); + // expect 8 total matching results + assertEquals(totalResults, 8, String.format("query `%s` Results: %s", query, resultUrns)); } @Test @@ -1745,7 +1745,7 @@ public void testOr() { String.format("%s - Expected search results to include matched fields", query)); assertEquals( result.getEntities().size(), - 2, + 8, String.format( "Query: `%s` Results: %s", query, @@ -1776,7 +1776,7 @@ public void testNegate() { String.format("%s - Expected search results to include matched fields", query)); assertEquals( result.getEntities().size(), - 2, + 8, String.format( "Query: `%s` Results: %s", query, diff --git a/metadata-io/src/test/resources/search_config_fixture_test.yml b/metadata-io/src/test/resources/search_config_fixture_test.yml index 08e713c6b1cd38..e3c97c267188fb 100644 --- a/metadata-io/src/test/resources/search_config_fixture_test.yml +++ b/metadata-io/src/test/resources/search_config_fixture_test.yml @@ -57,9 +57,9 @@ queryConfigurations: boost_mode: replace # Criteria for exact-match only - # Contains quotes, is a single term with `_`, `.`, or `-` (normally consider for tokenization) then use exact match query + # Contains quotes - queryRegex: >- - ^["'].+["']$|^[a-zA-Z0-9]\S+[_.-]\S+[a-zA-Z0-9]$ + ^["'].+["']$ simpleQuery: false prefixMatchQuery: true exactMatchQuery: true From 230bd2674bb10ae004a1753a30e9e3d0e0a573e9 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Tue, 3 Dec 2024 20:01:12 +0100 Subject: [PATCH 119/174] fix(ingest/looker): Don't fail on unknown liquid filters (#12014) --- .../datahub/ingestion/source/looker/looker_liquid_tag.py | 9 ++++++++- .../tests/integration/lookml/test_lookml.py | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py index 7d4ebf00cc06ef..f48ba6758564bf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_liquid_tag.py @@ -4,6 +4,7 @@ from liquid import Environment from liquid.ast import Node from liquid.context import Context +from liquid.filter import string_filter from liquid.parse import expect, get_parser from liquid.stream import TokenStream from liquid.tag import Tag @@ -81,12 +82,18 @@ def parse(self, stream: TokenStream) -> Node: custom_tags = [ConditionTag] +@string_filter +def sql_quote_filter(variable: str) -> str: + return f"'{variable}'" + + @lru_cache(maxsize=1) def _create_env() -> Environment: - env: Environment = Environment() + env: Environment = Environment(strict_filters=False) # register tag. One time activity for custom_tag in custom_tags: env.add_tag(custom_tag) + env.add_filter("sql_quote", sql_quote_filter) return env diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index ab55321a4d7342..4cd2777dc7dcad 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -889,7 +889,7 @@ def test_view_to_view_lineage_and_liquid_template(pytestconfig, tmp_path, mock_t @freeze_time(FROZEN_TIME) def test_special_liquid_variables(): - text: str = """ + text: str = """{% assign source_table_variable = "source_table" | sql_quote | non_existing_filter_where_it_should_not_fail %} SELECT employee_id, employee_name, @@ -903,7 +903,7 @@ def test_special_liquid_variables(): 'default_table' as source {% endif %}, employee_income - FROM source_table + FROM {{ source_table_variable }} """ input_liquid_variable: dict = {} @@ -958,7 +958,7 @@ def test_special_liquid_variables(): expected_text: str = ( "\n SELECT\n employee_id,\n employee_name,\n \n " "prod_core.data.r_metric_summary_v2\n ,\n employee_income\n FROM " - "source_table\n " + "'source_table'\n " ) assert actual_text == expected_text From b31d849b9f9f6a57fc73d58b881e6ae955c9aeb1 Mon Sep 17 00:00:00 2001 From: Jay <159848059+jayacryl@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:04:54 -0500 Subject: [PATCH 120/174] feat(docs-website) fix links (#12019) --- docs-website/src/pages/cloud/UnifiedTabs/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-website/src/pages/cloud/UnifiedTabs/index.js b/docs-website/src/pages/cloud/UnifiedTabs/index.js index c0fbc25a8de6bc..d17138fcee629c 100644 --- a/docs-website/src/pages/cloud/UnifiedTabs/index.js +++ b/docs-website/src/pages/cloud/UnifiedTabs/index.js @@ -11,21 +11,21 @@ const TabbedComponent = () => { title: 'Discovery', description: 'All the search and discovery features of DataHub Core you already love, enhanced.', icon: "/img/assets/data-discovery.svg", - link: "https://www.acryldata.io/acryl-datahub", + link: "https://datahubproject.io/solutions/discovery", image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/discovery.webm', }, { title: 'Observability', description: 'Detect, resolve, and prevent data quality issues before they impact your business. Unify data health signals from all your data quality tools, including dbt tests and more.', icon: "/img/assets/data-ob.svg", - link: "https://www.acryldata.io/observe", + link: "https://datahubproject.io/solutions/observability", image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/observe.webm', }, { title: 'Governance', description: 'Powerful Automation, Reporting and Organizational tools to help you govern effectively.', icon: "/img/assets/data-governance.svg", - link: "https://www.acryldata.io/acryl-datahub#governance", + link: "https://datahubproject.io/solutions/governance", image: 'https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/saas/demo/governance.webm', }, ]; From a004c9293d68b4e04e6d1024b74ffc27d4546336 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:47:08 -0600 Subject: [PATCH 121/174] fix(ci): fix datahub-client validatePythonEnv (#12023) --- metadata-integration/java/datahub-client/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index af71227809d2a7..2535d091f6ce52 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -62,7 +62,7 @@ compileJava.dependsOn copyAvroSchemas // Add Python environment validation task -task validatePythonEnv { +task validatePythonEnv(dependsOn: [":metadata-ingestion:installDev"]) { doFirst { def venvPath = System.getProperty('python.venv.path', '../../../metadata-ingestion/venv') def isWindows = System.getProperty('os.name').toLowerCase().contains('windows') From 82774bb65eb53d4b2126b08ff343d8410226038e Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:48:46 -0600 Subject: [PATCH 122/174] test(urn-validation): additional test case (#12001) --- .../entity/validation/ValidationApiUtilsTest.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java index a2c9a15d92f90a..2ab6a50945ba37 100644 --- a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java +++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/validation/ValidationApiUtilsTest.java @@ -79,11 +79,19 @@ public void testUrnWithIllegalDelimiter() { } @Test(expectedExceptions = IllegalArgumentException.class) - public void testComplexUrnWithParens() { + public void testComplexUrnWithParens1() { Urn invalidUrn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,(illegal),PROD)"); ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); } + @Test(expectedExceptions = IllegalArgumentException.class) + public void testComplexUrnWithParens2() { + Urn invalidUrn = + UrnUtils.getUrn( + "urn:li:dataJob:(urn:li:dataFlow:(mssql,1/2/3/4.c_n on %28LOCAL%29,PROD),1/2/3/4.c_n on (LOCAL))"); + ValidationApiUtils.validateUrn(entityRegistry, invalidUrn, true); + } + @Test(expectedExceptions = IllegalArgumentException.class) public void testSimpleUrnWithParens() { Urn invalidUrn = UrnUtils.getUrn("urn:li:corpuser:(foo)123"); From eef9759f880b74369567ba7f297ebb0406345f6f Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Tue, 3 Dec 2024 14:59:34 -0800 Subject: [PATCH 123/174] feat(hudi): add hudi platform to the list of default platforms (#11993) Co-authored-by: Chris Collins --- datahub-web-react/src/images/hudilogo.png | Bin 0 -> 75205 bytes .../src/main/resources/bootstrap_mcps.yaml | 2 +- .../bootstrap_mcps/data-platforms.yaml | 10 ++++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 datahub-web-react/src/images/hudilogo.png diff --git a/datahub-web-react/src/images/hudilogo.png b/datahub-web-react/src/images/hudilogo.png new file mode 100644 index 0000000000000000000000000000000000000000..4b58cc5a34826df31e370f810a93116f033c5c4d GIT binary patch literal 75205 zcmcG$c|6qX8$bL(LJAY5MTjDGvK$q%j7mffqEn|0sgqWFjRrH5$RwetRMu2z?^J}w zjHR@aC<-&9&6b&wWiV#ux$c?KIp5#&d;WYn=hbn>XYTvDujPHM_vfUShs(4nbEaSz zHqCXNlQ)Jbu`x_>rSfF>0d1u65pjU>MN|!zghWHU=M2x-jgpIfnK4W7x_J z4ATzDEZblQznC1b-o*(MqknQ~1!?e!QiSUtYn8gDDrhNd-_Ji>4Ijw`hk0z2tG`0H z1S?^#POCOXe)=IKCw=8#)ElvXzr68Uo;7RVnc+_*y^(uT(v#2FSS|HFyZ-#s*;W4p zw0MNCP*V#Wx!T=7*~aYN{<^W$Npj75mQ>`AG>0{b6}6B5p>R^|$=ly%I$HA+^1drR zTXDAi{nTh1>jS|}Q#E$y*L-(hf3w~6WY749KO5h_-|?P)qh~0nXw&PP+ zg!s>qPG6_(ml>W5lcYQ<)R(h!^!&hSJ4v-N8gHEPMM~k~)tLI*v`4n5aQ#9PkzriF z^k55(C`z+ZAK7lhg&v(OUNo`v3*hbK-F}xGlCLRZxLx6du+PG%#`q0%?QjMX+!G;OXKV`kAMlK)H3DlP@lVJhN%>U;| zpoKw24l!8;_e`oPuR*E&<8HNoOr*Xl>dY{g9g>j_hjfNzhNzFv5tB`(%N}=6l|EiN z8^^t#y~}xmRe=kDRUrKvBbTqrj?UHsN_Oq;^H^fw>t^rri@;se845{KIqvDP30p1W zpZN(`IcegYE@~6!JOP9xgAn)zpSQx}?y|=t%#!UBYzP8UGw9n1M69mYr`P1naDGjY&r|9M zsmEs41T_oh!};?j#L50k_Bi7;P~)-a0O5n-=S@8}))q1`oBxTHdHE^pGOj-eKG&T1 zJQP0ndH!i0{pk4w+2J#kfpU`xc`9id%XsMV2T(W#DAd6P1BFEss)DTBI(|Z+N|uJQ z=aWHItglTdZslv4pebv>Vm*Lin+!j{ZwgOMmu(vf))L|d%g@5GJnfZAWsIRDYBCfu zSAF2vQK{>TI`s*%lX|TJ$}J}pS1s%h8GmOW#vF%TC5n5zN~SQ5Ct=q*{R4!V2A}hr z!oAkXG+@zeIPC1dWy-4*FH;!Gbb#2*4N?H{u3xoJX1o$J4@`e#yV>BgQc-7a`UExl zK#dAe6NL+1I;4|+Np>iSts}6oGVt}VS2{OwC>^j3cgG3!UA0ig8+~KYN6oX3Y}E}u z17s8~lZj~ourdR%VM_{iTTv%PSN8Gi3^1kDpi6t4o?IrXS;xtC-2-03d_J&J1E+T_ zTOnD74HOd`?CSM_^U&b)wWe^!1jP8qqFv!o1cT3~;d2w&VcAKt&!vZ@Sjr?053uEo zemg+mtgU$j&&N$XpMjn~59=6y{sy0G$v*d=3OtSj8xC-a#t-S3|1IPFEKr<`SgBj^ zu9vekkSY!ZSns~}1Lu}Y&F!XeM|s)Pi`T=U6lJ(;ue3lWDT&4VfW?Df*wAC6?;?{( zdN%OJZNgj?!q!Rk!_frv;5vXZ$Ejv6urgCq3MgK4K!7GMK4mSz^&f{_kI86Xv7@x8r5ox{HAV0%Y#Ez z!iAa)>3B_mWY}6b)ZZW}2`~WtQLWHgMx+mbWX3TdQWF<^HCrLlNy-C84e($O*uovB8g*d2j+^Xr zCm?i1hIAN3GM-tVh0jmG z^Q)Y;yV)y+7|EztL*xKvNwpX=a+Ymth^Pl_`bLd}Sc9=qw#;JWC)fPa;*6EjQ#mRk z>OuQf7P+<^Gfv!$f_Pe!DJ(nviH#ZAT-_mh~e{{0D8fh~5GvfUg3Xw5IF=KhpP6?yn| zz^@>u8a}LU?kIgcWC}d413D%>DV87uR-H6hT?yh1dxIcg3|m<$Y>EXCV*_US3fB+b zAM=ncV-6^vs_;vw>1)v~z9R|L7Ahyu3m~9=_>61w8?H3XvU16SLG$3fu!9!JpTOK)1`U z<5Om9J`3NI5dR}^el7|j*~?_#v@-Y{1r)7YH>Hh3H{D^#C5KEGh3R!X_GB~Y#g8j=MOJg@`8BeP=`m${Eli3e+6ZNQ6D z2znxNj;P;C)Ru6ZrLCke@+C8uW&d@w-GbE|Q>DET_Qa+F2z>?hlZIY$Uel{aV&}|d z&`SoyRO+kyf&uc+{L9*;!#n{*uJF900ccJp*!OMfV}K9JS==*i#$&dk)3PGL4t+Km-}rj0tCVaXTekQ z*q3d`Te&fM;!3Vio&1weAKdwaMZwppCX<;_wolqc19|)e{P(wJR}WSp8a7@APS^gz zG{>jPUo_`TVJCRLXf~}V83_J-FY$cNVG$ocwoO~?8?lC8io=uM?mQ;owdSOZR8-eB z66hVOzrjirxA0h9ebx$ZbL5v|P|lMYd>b-14ID1`klCByYr0onnu;3r$cFnL_0Q!DXp4s9#oxr0GbQO7 z%6h_XHQOUsjm;q74LDbIuiQ$0G^%SsXc~_46NZzA$n&BubA&EmQftSlwY-_*g@256 z%vO)~Eg~O6YMEgVdM3JE0OG3Bb*>5HDsEWtbEH*o9!<#fiTX{D;?SQ%>IC!NgARGa z6$m8HEmFb?@7Y2=$@f;@V9|b>e{AW%r}21l$k`lro2s}$Y*4n1lhb9d)=JS6RfGz8 z)t7mKk^w#*nX6+wpnTXJ?eXXg(6A}Q15x`(_Cp$f68h<+$RBUevC~T-iGu#cDI#E;`UDdMEyRA-{Z7^n&^2zV+j2ELODMk7GG0{g-*w zt-R_T5|G^p_>`+p%zG^fg>qdb*Q&S>TzQ6b)6E3ISERe z__yH@hAqtJ+h-{hbQ%T(dAD<`=ZIgFh>rZo%B5@Md%jIkW!FFL2<*zbf|N$z05CZI zoo&kaPVi7}tGQ8i?z;rEV9v|;^k`1FpU|fW?}7+?37L9IY}X>{ynF!wKYv_jk2$Jz#nfM{ zWr@0a8$Xe_H}ONIdxSkh#y;dbXrFX&9l8DzeXzegmC@Z5;~BgM9il{Z>Y8=5LIq3c z@I-~&GC7*8D!Nx&s2e$#i@rAlN<)jTBQ?}QJifGtFSs{WKoi9X#Giws$5l$N_AN{R*P(IzK3SObUx{^NeabvW3J%-*wLc z!ZIW}qYpZ>>=b-)DDYCwP4e)o95-PpQf4m)IHi0~mIss_)F|=|f^78{WvqndK|*cv zQnjEb>OnNU4|4bs>-`G8+6ZYRfn4GNdk~spzqvtBPfSjhp+`MG=kA+-1TlXJFPbg= zfEUK&HNg#q?i^>EjD?2|nv8WYgpL^gdowCNQsf~*(rR`FfSeL=6l{LUkcLkBegW-n zKfzD@s)#%d9>NxBgMNh`T1&X8NolBfKwHLgv!A5oaZjq#ZjXe_&Eg+^6GDakoBiF3-?o44Fh~IGDt2*pQWkif(d? za5SzW4;*2PJs0hIvpJ^-Ka?p|APueS*2HzdlPqmMl6A{@14LGTMLM}i9Y-}FrpIe9x_K(j!j2(GN+m> z<`?gtqnH~0>W~Nqsb?mAH^U z?4l4{l8`Ovf7_blRHMgdvbt$~gG|vw{%|0!Ka?vJY45@7W&ho*eF(P1;zLcroYxw| z5x}{OZk!uwtBva)A&?_)QX#;(t11HBe_AX4PAJ)BW}d^P_2uNGw5YzfU86V9*rNB) zC8GYV%a3?Yc&K?KSG;mcJ5r@7&sB&NpE)a+i}x$DwJInVHJ&mZG zT(E-Np?5WV=X69sfQ$e&Ai(8bzgns)ku`90#o&=U&jGwol(Cgc+5~fXU&`2m%LC|d zvqXnms?2974son(krq@rai<9sySD;g{k2?@-gQ+81bz8OPEz~bl1zP%g6EkS82eMI zh#v#kHSEG#^a}hjz;qo$fJ-1$^6`7NYe$p62=AGW#LVTDNz#>{^^@%w6v(>kPy`-n z36^s-mrwJ5`A+EIE&*b!cBg`_z5xWc&Q7-y-pt{LDpM2BA>`PNlO>^Q7}TvcdIRFlR%YHU$X22#3|45r zhA?Mv=pP{E{as)3bx~8eHstGwbtDzoy<3sn3nd$kCkkKBG5OIqr`dlu2)OeD?F*)p zU;fnL$u}kd!hTIs+vIWfu94J z?{aXR=luk3?$=dGO_QIPA}2$UUkQm2#Z^+K3WbuQo4qP#e1&fx_$+e;i!hUxh)^Za zZ{(n`sFT$B1wTT|8P486$UHK+g4P&CCUqSW@2YdDsV8#DZQ`P+V90f*_K3&04#X>b z`g;zC6NJc0&72mQ%LAI`Lz&FEMt${&nwJxirF!EA;wpF<3#pg+dJ5BB^iWb>+J@^7 z>)C=?&!BOihOh_6zp#WIw{QHC+2DZZvyUDPIjRL+NOEp#Fm)~OyPYvh449Itt{i(K z9Qv=m)x&N*vw!A>3Wg@U$oN}1buBk|)E;HHiWEzTD|`&cScmB;Ax_%SVj<{cXB1cO zgCdJdX;zekTNkP1)vk!wb|FZRG!-gT`X#^+3rW@K&D#MrubHr@2Vc-3T~i}Z zLH9W-ZBfBcp>XsY=8hsUTW5Zh9WvB=LegX04qnzVzM&d7mP>xUa|dsTDF86wAFk+) zMbYTGSCj2hmnd|8kLP0F^7%Atwan}ks!coh%km)Fha_6sPg(p9otfu~o!{l$Jh@So zA4&MF|A)4|gSf>Na$`XKw$oIuOU+Qz0A@Y|eBG@Nmg5mWSU2v#x3Zu8ETQXw&vl6r z+?zI1`Ns%Rr$PlkLYMfj#fnG~GJcn{OHB$Nz|H|+OwuI_^6$37H^<)h);Hx4IROwg zBf>$NueXEMh6d^8{tj|X;qWczbj9j1!|*qhNRN^Jjzg@vO%kg@IqulS?ySpxr>S~z zu=Jm3=?HpGrc}dwAfU5^k`9zzd{M_|CbkE~kvC!}_Tv*hf|mxFEU;6Pu!k>tTuJiRK2 zshacZZp^~E4~pbprj}^~rj{EGpy&pmm`$K8MnTRlkF#9cz#xI}$NxGMe4qV$2%M%y zM-}iz<8NC%BG!yhsX0ST(tqrX`&iI*k}3WU(OfyAhp`bHixE^PHs!YtnMNxW>e9Fi z0|YU@|Hr9h%$v(&I6=@85K+RsdkrjL0dRQpp{~9*WMfNPxoI2dMB&A;#(MFo6!9Bw zP;KQ>zG!rmO1lkip?1)!QxGh;%o9!Fu5P~CB`@5Cf z*w7nG{=KaA$ACi0GioinHj`^#H>5jsthNpQPh5ZsiTZ~jgm$)6&Z@2co+bi^KAu9Q&F!^VX zj`yWh1PNlwdN$w1`;dD|PeB0*{`eCo@fd$eT*e&&=<^@|d$AMUBW+iMRr?8FN z3}Le)rz^%xqn$gttVt37q>uE@bcC41iRI4ysF*X>jH;Zu2jOu;=!`4`c&hqfizPG) zd{>vT0zIX9)=@AyDive^fsUCJMz=$}s8+@b56D#|MZOYV-NDOW!9FYjy9N4`ghcS^ z(9zqswgyViCXhRPJquiUC(KTkT4~sH#+rD5m^ajxMCt`Z^#&F z^MF>9!5w5TZNLmim&|M~-mg)ZtvBHwvy-YlK}^vSF}>kCNIlbaC{NFLYik3Z-U=OS zC-B|^c>fNzfUrZd#ctT*t0Y)DxVC0}QJ`F|Iap3@NM{L$uQR7nkwt}&AmxD^7=$E< zQvqh31ZM4rE(yvZ6)9B^=7p8923)1`vB_L-kC`7u0yNl~*E^4r)*$fp-&MxaYY=Ap z(M(5Jqa*4hZUL=e4z6K#B;FvPfPaL zOCS)e5V_VCb9r9B!`hYx97q=_X}}J&_aD-|`&HO^PH-4#?acM4QwRwovR-}j=L=f( zmbQ}Wt)zDRbaVI?jl}vFN3pstS@K|=g@dAp*~31vpD>`;ljMixkoCl`)J5+-NdpJO zb_SwF2BfyAF_$&LB}x_T;tWx14x|mCKeBFdbBee*C3KM?x5OF~<O=!GU1!-XMgc*sIP^{ORsS0 ztg+$8WC@Bv-!(x8Ec|lAG?v7rc#rr)?PU3o#_JP({II=kB;lzdNzI5oGKrV!gYXW= zJyAixjMi|Yj?S~T7o&_0at|A+bWnKL4VwfiFioIPk5Q1y*A0#_!Hd5KT=!aRRdV6G z-c{Y$kA;$=i9%ZAe{8m2nLXcS@)!#F6)E1MA0Rz-?wN004u&V}JWRjv79yvFLlt@- z6z~wmM-I@WxQ5!$^Nj~-3g!!erx0n`bV~xMMNm%sN&#xGJ8`2*rx6bKlFaT~?BJcY z-ljdiU8h|VBtr9>JwMNvB_MW1DfzDijZ5P2!x|;OOC`%ku7ft{9VhYML~E1{B`FEMd{Xc8}gWa1RyE23u}7*QVsii8=BLf>hU6y9td zz6UD5`Z5MVC%!rfgct@eCHuyb=fhEl{|`sNhBRN&F+EMSf~3c>2vG0L7Zev>>f`U< z2i2BW7Gs0suAU9eU%I3f$2Ik?%PuXUx9q(${soGz0ErJ(*gGdnqj{JLk;r2|FsuvZ z#$~Wfkt8W8>NLHct>DqsZ18#D^!JyYKI+J|G%lbdptf~90^V&1>zaU7$2kOp06*<< z-g$xRzjcYalgnY_S-AC37TpTz6u8SvlX&K9L49W7z{77vdcB;+opDV-D~qV~F+ZMH zE-8JuL3z~&nsioI%122kc23U*Z!^0=sGBFqlB?~C%}D`7t$*6EgBPQ;f?YdaJ_Q%* zFQC;+8Wz_U0*T<Jy8!$u#3Ts1>sVi(_ER#QdUSm9I}_q|11>1{!l!n24Uji^jh zGH#nVz&#Ti##OA2(;OVgTRYgd71EXOcz#EQ} zRkRXi3&bb^h8{=stcMN>vcBw{(Dr43G;^S|?)OK*uWbMdI$$;nn$XO3_Da~fe12A> zG=|IiQuwcaAAh-WJvB&rb`3_g#b6bG8-!fBXAxl%UlJ(DO558;yggf~{%y+2wE#Gx z#|2#p;;d^|>K;$KfBHwPr|>}K;2(P@R4g8Xoiz~guSNj>2*D3jA5SW` zb2ELRR}k2fBdUVw9k1DGQ{PB6c&~iqECX0u2~Ls5AsF4bKmKWR!0)n{&a%6Y?;v5X z7L1-vh6>tqIU3g~jzg%sBl@A5^Ioxjs=MKDQgU}fIA9FQxn@MoaVvm?zY{+Sp1jWf zzlL-JT5ajYEWtsE9j%j(RPM0FN+F*s``7rgT&3*q?#xMgkaa~*8wu`EY(&0;s zgY~27>5%9x{)20qA}#D4`-z+o#8$b{*Ps02I*#G9uZn<$K|;7T$;O^VuDCA9a=nQc zR0Z~W+VgkokP=Bv-vyzb4eG%ae4Dj?qzB5i(tWI9pY?DikFJ_nq8f4k-_6pxD z@_%tB5zBU)UZIc*$>YsKkC$xY+xl2uSFON75db_JX}h-FH_KOEZ4%IlH=q=Rfd+h~ zG6};C!n$SMv4PX~{7b3^!^Z@X3LYpvsjGrnGVc}$eQtvy`7g8a$GRhp5y*E`@Vw(X zicytr5sXNkL;B$YGHjJaHCbCEni#dJ%x5rj-IZU2#sZ-lK&T;+Wy2V@ec+91zCE7! zoNL*~tK4SCN)1$F5-&X)0=`TLCg<1M*z4hzw{l;5qbhkaG^{*{i50poIalxz;IzjA zTz}EtrpfTxk7#%1+A5*qg-^hx32hRAK3^zj z7jFy#Ef}n@JljY3Daoj>XCE7XF$%qkY(ITi-*GLFbo?}ur5R8&*P+y&FO95hk9y0p zdi?AH+qGn<$Q5**DpKw!mUzrC7tAvc!UrOCf_1+F9b{Ss$cfl_N=OH#EVN z&9q>b+|`qKTAR$G>F1!DISb6QDj8gB8SBMO!!ovdYc&3r5~;~(ocBE{$8=!%%Kl|@-ke*43;nyR zJD&QNPXC%`F|3D&hIA8PXj%OPo)j2<2-3eriog>z@`4nd416hs8hPEY?a}7mSg|&0 zh_xqkjzrZkCr3|x(g**N+dLB>!4Wq)*VNMZ0_~O~l9J`Nxsl0(iO|I2k2;5c8?e26 z0BZ6Y{!g64|F$TrjC8Vcr~l4tw>sf4PV6~a&#rl3b!o|QR`jg)+=}2O!}3C<EZSRcEf(xvu7lKF(!-Gi@&0( zFaAf&K<#~eNJ@)djX0A1#b8)|1ka(hpQK@=NZSg%8tyaSS@aZrQZdMlto76ukMG)i zsIvMigpR8BwF_BYv>i!7_ecv3ObfEl_{^T#V>Oxe>ADh@0XP+pBHadI0AtB@x#yR) zv$F3N=dXa)-9fs~0zH{u@Uh~cmYvlS+=tc~XJz9RV2X%y6*iJV!WjAazUvQ2`FnPU6{7|@wdmmT)T@FSw z8EM;SycokLi|10t8wAN5f?IH|sxRgjwh-EM*HyEasRW_F$T#Bn zXjB<|`5#UAGRjrg6x_k|jRz~j|HAxK+qvn_9krwB&8WSjcY36v(sTshO+$4zy(fBI zvDysJ%`YVG*k^C-k;R{@TU5VWRn3mVa-AmezCv3?I#7!-=2twTEkkDe&UP_{jKU;{ zU#0j3Zv~QJ)kDLD2?aY*|M+ocS6wjyN#`Fb>hWrheX$L{UlMHoDFAIlSrL3T(J zrAeY&m=o8@JjR4%k+Co4+MDQYAXlD6NmKhjOUOx%ET&*hmFx|)Ke-J@_#M=b0J!fg;+1^7Hf^xNtJL>t`&s=Jr)qVd&Yf_Ly z<6Oqt05s0l^vP?(@nxOgw$*-|0)08j<%_-6ko4;uCOTZ1ThpwEitW9mf`cS9o95{N zeR4rH(@M#+fe(|0po((ik9`J@$MA3-4W2s`C1Ow8o?rUU3}5Qh2ZQEikRJ#naG(1pj z!X6}c7D53*VOICfi1K!P321oad0q@7st9jlee7~=(Pm18G_7MiToyzZU2C!oIWhxg z7H+;%ReN3lgJ6^x)3yAy7QxW*0#v*WiV>7w8up-J`<8X~I+$0qeB@auLE3iAV5e@@ zbFK+?syPRz{^pZ;Nw47CSlaPjW3tM{^8}ob`>3Url!7LG<`%O|9>ByybTPCpJ)H?R zX>_5(h=^UHq}Q3C+eu6rW9+)aZ{f4vzZJ9_6TQVmt-oeech!T_f-%5J?;kxV9o8?? zeoBch)4`d!)Re=VJIJ|V@13@N%QnySQJ;)yrD53%poj;G43p1n3fvwT1~WnAmHT`e zKXgsBpko%GNI)5@DE3)jGd+Vi+2igNu)RN{JLE>+x!GeO2b=Q+{PQgCq1wB!vz3p! zBBteXh_E?J_rW=Y_kDkrL6FWqQ(_Phv#GTLDq2uVz9zc5GcKWlCgSN2>3^+#Tilml z#}~Cqj{WG_F7e+zHalmMe&z_Tk=ww>=Tk>EaJylMAPDM~d!c|S5EY7jpe*RhJ!Cw* z@{2&ogI|j5hR4|anEtZfQy(=CEZZ#Fu+fT=LLK>Y2k+{;)~iCe21BuoG9wE6%f85= zIoj77CG!omuIwZ7?})S7&E{-|a#P8kQFYFrUXNo}`O>-u?P3h;wu93Y@!y*ji@x%+ zatapz58x{dksgck^Gc4diWcu03kLfFQaClfn7EPITB%R%~Av&d#c=FpwCUR8hV5%f-r?31X$PkrfvO|i3A zc#91f$`JbAmkfbb5pG;14UIwCm|YHoiyp64j=&+CK_2ECnghf|W939RBHYKZM7J~Y zTk|GewV3tf?sdDnG$==!892BLRpr-!WYa&r-DPU<9sJ~4RBkf-9PRX4-F{Aj-3k2n zXm02oexbVEceSHa#U-^We?Ae!WM~_tzJswTN>M2&RjtTew5KNzqrlMfYnZeC&4sU? z$N2HhC{hpU#?;7JI8caH{jyf3$Bg&?_AB5vKwZ?q$~4Jp7rPSrxhsGhU_ggqwteF% zwaDwHzivZJA)sJ}vtzcig0U)P*mv>^ztKMe< z2im;>80ymp*0_Pvmr2?RFiZwP7JFSWOZ^{3Pd7A9dA3wbslLhVy6zM#!%ohwu^88m z-!9*{naYk?w`2(0n9U?4ujkw9T$xCzhu$jUL(5pTKdUBr?}K^d=kXnNBeqa5JBMyT zrp>1W?nl*h>zk7wct2#*f?*_a<0Z&S3Ws!tcO2*Tk`nIx!h&a&>=<6_fT3>ZF0wLy zG4)p(aeL+bfxQ5{0T4GEZkJaWE{FR9a90DkdK!Yk@fsC}ZKp81RJNU_?HDZ}b}wZnpwk$Lha@EEozo#gjKa|rW-%YjCR?90fDCK>ha zbZTSzUh?~`<3kXV+{*;q{Y;|r&wcLvDDlyDu6Y$yks(e^X@xG@D6>G7v@Gvu|Hs(C z??+ZN7he%{dQ;MRatc{JS%pv-(7-}cvFww8_t4IlT-bq*&=T-&y+ukFPcV(dFwp8X z8|+vUR=#995)BRh#lH6(r%29x#H7r)ht2|oelSBF8#SN=N{4HppRKm&BL7P3K*iQ$ zj1Z(7Rlz9TSZ_Zm-lEOowqRbe`Tv$2qT*f{L!haJ#>La0J-P?$l^RewUr#2oj=dN$ zFSF-bt#2A2r4KbJcoBgf%XV(avK_pAAZ$&u?f%!{a9g34?-m%cgp?0@;{iw8cn-^U z-Mf~r9?AG<2iOEN;eIAN>ji(&ZttE)bw9-rBOrqpWAylfK z@r25&$rIEce|X2G<}zsiaZt-gkkTM27fzS63w+lbL(hiO9^)GXgi4wV`YMM&!tdS* z>RffK!SRQeu{uU1Z5!WIj+QW~g_NUg=I{SOa#H5Ct`8?fxrA(yGpJ@RT;s?DE6Sj% z8-Cur_(%{b9sKp8g*0yF#Xf@d#j!`)(5013@6A|X=U}^y_j)VnfaA3`7zqGy=4Tct zQMzyRaX65G%l8ab1jjcHgv?}}r2cuiH0qxtwPF(sa6uplcl$}B%Rk#47AxKma&P)U z=EqfKckKSJr*InI;57c2KbUfY8KfRvF18SsKol#6HE8W`8cO9Z@Uu=zikha;32D+vB1zFcfAvASXY$g>J%>P{9RJ>t%sJe6 zn|X9>Ns*E~r(FEaG352rRSv9&v=ZG+Z!?i8Q6U;N(3msqwD6i;=G9aHI zLVLvTR3hLlh}vwg|9Ld?yO`@8QQs(rlyzVCNhQhyO-8r@TGD|^Mb`Vi>G;>}!0z-| zCc$Vm-!}8K>BvRgOwVi}X&)NNR`5*>@*H;t+_nVA=(SNkGWmA1paQ8!Mnc0h1!Mwn zDZz+KocoIJQ@<*Y2GJYYYDnYjw%eWexS#kti&-77=e%Nx!gSiA^@jSYSFi=!cwbg@ z@P#F|gMFdmH`>{K{Gy*#llNR{E+2>Mw%Cd7DOAE-Nf`IH4cc{?e56N!eObFBqkpi3 z$-S<7)&GN!y5r97n7{z=KD(tj)@=qI1ecR4Cktw4+C69jBxNXHfP2~`4Y20luIegd ziH=zPeUy?!cxuSGJQyOa06&s@4;; zl>5(Bc36@zrE}dcxK#|rAEf2D-YpkY1F~3yCBH)r*7{~d+x!1p#pz~9qOY|b$Jmg(A>oAaqkl>3u-I<>?Iafo6P|2Bwd?KZAv0~{tE>CitywX|kwtHH~t zm=*X^bzX+VDP&Ugb+Km}49iWBD>|dAosM1Vc3`jF640n zt{*`%A&%_R2!|^z;AibZ6IWfNgzlB7NB4$bO@y)8DpU@hHxf*rw1Zd0{`f$#zNzov z9eXP0@N$ZQW{rZQGJEzRdnk42%7^v9rLKP@6XNS$AiUZR<%gPXSg=u%yrP(u_OnW9 zPafD0MFs9AdF_*rRD;U?Wi(A>qhEmXYI5}lf%`*mb;qFY80a!Z(EY~oLG*)ydDny8 zwWAGT#t5(d9+GwWf?cZFTlPW-;a@fR_{cXfea0j-( zAb}4p1yfiKdc2pI+Rt`Ge#`QFI(njyka0#j#Z}F(u6ySR4aPjV=mw7l=-exsQT%TJ z_;c2laHi%S1jPWDPSn?>xPc%Jg3HvGAAX{ZJ5_c)tpzS7ltK`8(w5)WOtK#zRoZj? z-=H*Ee!A#7cpxdMiKex_8e~H-DD$QAk-Cz6XwYt|;B?=$qpii==-hW8LbwGH0{z~Y zNOJ=o9KU(trDwOBFWoPBL25D?+i;mzYrhsh&LVZRkm_fc#rc0ikZ-O4b`}M$IzsIRPKZAGkxPLAWKTLY%aZjhKPk#E}Y`=wWXcg z`JH}|xT)|)p6N)(A%5DC%H~}W+R?1%rXx-CnY+kY2kD(>w4KbCL(Oqp$J*zPL*QZb zplF7mCLW1!iwe6tCmXrG=O>Y4WoZn4dr9n(Tj#;G^#Qzn_fC;~u!=c;U$Oon=ohAc z=7M3sP4Jvh!Njap<)WY3*sG_2SN8h|!OS1_%7UTT<^;I+J{yjk$RRw>I|igJMWm&C zqL@W!Kk4IdAE;8kjzFCf2XdYZJlrJ1+K@T&ksqqRYqzCP=c3fS>A1N!AoYSgepXR~ zP;E3<9E#JPudqE?f}1GLv;Z-OOqU;-5>C`LF!)UCI>CjzSyg|5 zIi|)j2_6Tf&7tSFcSTgz*Ftx;Ghh(v$j=1Frj1z830DP&?+bk=`RZ3LlbqBTVg)|% z>9CN8X~km|S2D8`s9nP})sZQT!?A7<7+n2fivokrne?$YA#%!L`BF8^A^%=?B$2co z^K0@MtZE+z177wBR3_E3WHlb{HO!~ny)(qquHSf}f*(LM8^T^EF$vedSM>-~L!q=a zFYLN>nr*%(%z7khyO}RVaQr;4k~A{6C_$1!&Vc;Ns=E8=jo#|FeBX_5LNr5Yb5436 z1?)fi#{Wl~>D#Cl&8x)vV5Q{@q!bOp2Nsypj$}#PfoceDJv%Y~46Gsmb#cpDxn67e zh5-#*daFKx*;(ezj4dkVy|h&=W6h__#{PaXhNI>mt(??|axcd^nih@^E(5$Q#?Y6G z9=es#IIyvC8;J8%Eba3r-%G4S&Z$L<%s@Zhh15O(6)BYrJCv!4tR+L|15O#+?7&T& zKaKAGFs@M}-)|veKbYV&`VvGQi!U@8U+ zXm6i{#GY*Zxcva!?8tSTT>sAOs_taUeoe;aI3T5|n)-NY(-P!C(2%K3+O7zC0*A2Z z1jO43Lscog$MAn>k2|Q8v*RB>{aZDA-%*edra0j3bPz%_g15T7<6#x{&0WjO?#jpA zK&d9!c(ads)z`tgG&1-GBV;;5Ey2QKST*&=k;+z*nW9;){~XB`O2-TE(gNVKFVsXb z;J)w;7t9Y{xBw#uFz2JNr;FtDThCe3+Ym2iECLpNI>zCTeGBwcYv+2+n!$TpjM21vcp>PUJQTJXRw&H{;8@-Mj^Q8+aZcm*|&U?sR-dtgDAbG z$&Z|c!3x`km}`0)D<$Ie*m9wD_+9jpvKb0mmzjh;pWDLy1Vi_PYVkgjW{^+)4tM+7 zsVawFRnN3*ya%{Nnf_P@$ATKk*5VGij1_XxXwGbPL^&&O3KKNL&%=I_C31~{*ZEnl zwjo3j*o3E(aNMn@lG3>Ef7;w=I=;|y(9(V_NId;XZmjD8bluhouG@kdLky*luBTPB zeW-q>{XS&dh_4M85!)R!MYHox1@BGa`+s2Eq`*hki}Z^ZpNN0-4SbwyE4ujhVmnOK zf$<|zA(L37Cl`%|luvh6TvC6@LFFo9&-{#1kdrmch-6;m5DJr34p-)!lFiq9ej9*^ zt3AGdRn`Dh*;A?L%>`booAXzg=BwwvFeGas+JQVG{%9y=m%<1uE?IaKQ+w+VJiJ|3rwFkxhWCQa8eL znHc~nYx%b6f9)E5bhqYX85`i&^Bppe=}#E3uE?4lUwk~FOzUz(RTF?TYCTjKpaBsI7BJ0w?kN<2x{#h8`0MkMjxbaKB>Lc+!f(n zp>it^%RGrypm9HE$%`T~rgzvTpnHif0KUum{smuNfYYL^e@8LfO~zoxq<##Lu6QL7 z=`z?6JSM#B^7|Ws=K9N^-f@z%rQmCP-d8;Xfsgo77ZJ<(UuZBtb9XjQpzVt(i?4;r|e@WKoejf!v` zAU*`3;5Brg3r}1j0G!-iX7Kph_bwPn-0>N3$3Nk4YJ@e-Cf^LBJ0HWZ*)YKD z-Wp;BAX`Y4fUIny46m}fr3bv|RS1AEuTX#_Wam48;YkJ*v&T@GN`mRBPKrv@k~pqT zBQo-#sXDp*tc&l2YVW<9Oe0&laoYaoa!A^%r)fLtKGDG;M;_3#3VRNw(%=4vJ?7qm zVB%Gktw$HM6L_FaM*E$be^wmY2vT2)nz_C4IWTq$v#hb&6#-y+s6s`dQXk83?W*o1 zN;Hn4{TBq{N$5oj6cx!$X`3MpEfg0xfmgrIg~2t|n_N6@XaF)ug*`Vzdmrujq1f#L z&9E6Y^*^*|eT2!8^Czbv0K$fkLRH~o$XQ*AgIx4|36-h5tOqEjB(hwqhiy0dObl0^ z5w&WUy@OEF6-B)i1qcVEaf&{6)tF+2=A%!eYjO};SA-iO)y9Rw9rPIx)}LGnS8i4B zJ{c&owz?~rw-_c|V5YmPalqJqtzSyMXM^y*RU%CvB( z9!Y>{Hpi{W4Et%dTo*?WLu zbRNu*)&?aVK-)Wrkx(mr8euj5x8=7Gyg^XQrN38$QhRM6vM$bO=fd>%4qo*(j&WbC z-Xl4<1|j-u2d|hlyIR%K`X*o;Hmi556I$v#P{#P7!d5ksyrBeMcmuC=V_hs{P9BD+ zuW%NPyC7dMkGjZ7`$_7#4F@a3jX*`_;`+fjup|Us2bXRB0Q!!CPjg%Ok+z7^OQ$Jp@4>}I?v7%X&1AE;>|4kn8TTQY1%QUY zPak`I0;_QRg%s9?u;RlrM9P!~LEWwYglS&23WNVs0@Rzq$oZ~C3ptnH65DwT*%Z6w z&RVND@I}g4Bgpyu`z8g(Y!%kU8vw7HtgEeIVBhuMV6CM<8m#SP*#T`1W=-thB($Bh zTj`nuMtOZb=BEL1i>2g^7j^8N)|=lP1heyk$>ocQ6Y-jonvdjyE8J3fVLc9mmv8uK z+0e|5)FUMN7QPJq7~9z-t^2JeV2R(59^RFduV8~Un4bL^UFT2d$R-}|S9Z5FK zcio!bwSA8UuHxI8yGe_xYn*DvJH^?31do@hE38kU04rp1GhAJf#NYaudp1^qLEvYD z`sfNCMS2xb>Qg=kskq}wz{v9_Y|kI2E9^Sm9UENJu!C3okpdU(@aE4)=@-VDW2*qz zwxD_^#iv}V$2S2j{r{Q7v$+7n%rG1oP;PQ>*Lh;=N4Sfa4|nI&0Mppk-U!n=r-HI9 z24%SoFXRi^a~G@t0m^Cs6lmf{i`rFMxmwMA5*N3|3n6+sikf?7wR0g&u}=OKz3DxI z8U9gb3BLl435CDFeqVIa`9W`ky(=`5G_X1kEN2%qMQp)zeafIQ0ojl8;+yA)l(P*2 z5APJ=hu~h4Qu5M2Ay*+7F=pBoJVMA}^cjX<>5`J9G8Sk^sV}k3>r4+AOXg*z|H|^< zQoikjBe?<^Y4qcT>YS65U6%r2h&Kr?2sg(UWS(0Z=>Z$TVm)s_Z{dv(SY|fnPynL{ z{*vqq?rFy@O?7=MsN49Scp@kHSa=HdoQ{Xxn~>2fO2y78%J!liF1-`$EqYfpAd*B^zq4TA55jScfD|;{_V9<9^*2 zc`(-bX1T7w(}!G_e&QFEoUKj=(PFxC(dacS=OjaV*m9+F{!?fTuztSSM6&E7w%>gx^u_$}_&Q6VE~=$#g%8nfcqwiC zsM(pv;Jmw`H@3Mk(`Yaqj%Ag|`H5fqiLYhY@~sSx9C~88w3uyEgiO?2v-!&0Dwy$6 z#u79cov-%y$gw9$;$`T!>Z}1rs0ayi+gqWUySMP8wu3DcG>OBVg{lgI&TPMJ(!Jz!)Q3&+z!i+u*8A zyaLk6c*sHCE~M1U0W+Uhzs3Dw0l!c65nVRg{3~!fRwS9iOdC1cLl(Uv8`UirxfJ2` z7C)V7XN;;Q*xARr?Y<)rCCE8qIobeDlwD;XTa8X!fw)7z0PZaXZh9uWc%$a-gIKu@ z?ot_l;rR?gCq_N!&9eZ%y2_=g^s#*h&>e=r(f%QP`H1SHaBaBf zS%2IO)X5d8ljH9fu9U0LGl?3M#hQ!)f-$J%)Yz3Ta{;q-O7%qIgopdrsDr@VlVr%; zd4T-AhdjM}!>RVdc}&3_)$Ag~Q^{i95bgo`eDeTkz2S!Ccm)fD5c;!~`9S9dA9XoR zh?FRqklDT>vR**Li1Axiu_}q9Evhnm{RS@z=ou|qwvDIr3XO?$iAwb-`=d~+29(ip zh6=l}OmhY8c}W5;)Nn|{Tm``AHt&KL4!qrcbj{l0?a5n$c^TKFc+HJ^E&K@YYk(Km zgxrlPQ4)2TMp`e~#>;d9t;W7q-F6PY_FhuSo|Nl0iFdMvRG=T!a`xJ8@NlUaD^s63qrGl9Q$fT~81%u=F$=r*AYi4jAYBmuZ)pqbdSn3j~>x4WW1f#(y z2nWHKpIEx(AN#{R!e&Bb9`@CGw9ng$?xN!XoIJ)^R)2*r6WvpXuv(ZI{HD&BhQ_<&~ta4tBHviBQ=m@jg z#Ui;f_Qg4%11=IVy@MK!kFvVCz5K{*bWaCRWqd0%H(}molKU2T#PaJE#nNkndC!mM z!+NA{GZEZ_t5ODldk0G79O1VYD(**NbOoY8WmHYAXJ$dvrObk&0CZa9uixeyq^bMV zTPOf*21v_z`UTF1E1MlgRmnt7nx_N3Hf@mEe07}|SU<)}>mhT@VCv>UB@X0`k}xbassxZo znyG1^PF4MfFu*zuX!3zO^nV7~y!qc!lX2uo#&oAfAPNh_8~f7eA%}ChF+qF3#E`Ye zL(InIh*AzpUIL_5wH|kWew-3SI^#*utT~5XZmVWp^M~<`qD#y@kz2|&hAPr(CORw% zM>qb87P}OALw@5`e#tFat)~n7o`pOowU~^BBxY}f#PlKXsRkQOY46TCP43+nu zg579c#7L5hI+-IzWKM5P!DsNd8S*slSJy*b7KEVwo2;qrTt^S-EH+%a`XHv&@qj_p(6tVJs9z+E}Ozg9wEv zr)hCW){01kvX!mH*1n7!N=nWVLYpilA*Dj9Nt7iO6v4_3nM}d^A(=>R`s4A+XM6>(koBtQV$(~T*pK?pl^OV=aNm|L#JB)SDy2@!yvf%qCkPI}m zZuO#vrNRUpv1A~>AUOo?hk^_UNNj#Q;_i_AE~AQZwVt;vJ=;H^T6tfVo!(Vi?xdLV z4pWoay+$3W9ny{&yn62`qp-t9!~tbXF8<}rsvgNU#tE=E=8jl6Wf89)vF?cjDpDKr za4B(}0L0@vI%x&tRCPG-ZX$DrCI5en#+{MZ7~aaG23jp7t{u zh*mZ^mGbm+b#MxlTSL{9V<}vxiMv5{IR!R(bINs$MllJv&7uqVM{Y!fwh~TPF@zh1 z!8AE#a=33q%@Kg06d=(MoQ%hnh0z&G80A@-d-NTT4!2yYBhS{##M7nD5TuT5k;r?2 zgd4amBvOxW?C2M^3;9cEBC?xpQA1+APT33))S(eU)w&>8;kpm&Hyu+cnYuWYC;RAo zr-iB>hxw#Wju~AS&MPA_Tml>-8iNQ8JDf_mxaHiCf&}bB?fRy+jf~3(fetulXDJce zsy%GZjXIdlaM9cib^2MIFVEg-&(hOumh~9-N!G)Dxn~1LrVH8}HY9OJe>j2&VKN5$ zATy9p&Ue+MiMjF06=;c*7&g=jbZj;VqDNYE-wM~AueqNdq9HW%DpGnztoa*A9%=NHU3{(Xm<|<@-i7{h;uHAy`%0SuCYlD_Z5)pJm}CO6bM}+A$ykKK~6bqDyv$TJ{tKYM7te>1y1BdlDrpr3NFZe z*$UiYI;LAB!!l{0vWtMfCXnuuKx|8vQhMt(GcC1Fhd~Q5G_}u87pIQHm1Uo!?if|& z4({gEo+;(n=|dR{WgwJ!N1C}RiF@eC8nXgF2YIjm0gq%u9tl5v|L9vpHE^iHyJgSx zj7$;t;>-4=*$PmzWKMNmjd#;USt_mUrKUS5V&mU@{i{Q`Q;1bx?(&*;$`4QDi0m*7 z8+1uTY3f&mI?@{}j$&sEy)CWy-Hy=ZsH4l^VP|Sq%nf-34jG+mcX^p}BLmUset6@z z)FKj+uH*b4xO=p1UukDko@ZxE#WlA@w#jlSi8ujh)P0^Kr!|Zdu%Xb^qRCCEv@73` z_jt2(H}`rXSQ!Ool-W|TaglOAm_xX1Bo|>A& z%+j9EnSwg6wsh?-c+IZYob)hjw zzNPUAzr$BQu`F=#1g6R8dE6=0)`Ep|>a=vEPbVtwVmMIc12WW#iz8e6o%S71P)_*_+A%wQc5y+x_m79?67t{F1&}`*)lR9{iQuP zTcT>E_p*6iNop16gDexpjTbG#Y78%DcPwbIy`kr-x>4=yVV37CJYhgZ>7MI04Eg4> z!xdGu#kxm-h)@mQ-6a(1Ez~a-j*2)P>sx}o%k?d2>EWiz$?OAQ4r6vaN&n&{J!RJ6 zaTm>T1Y3Cb&ZtsxB}Pxvf%5Tba$RZVUA>nm&s$3D4#t=ATAzkMZ^1(2*ImkC39rSa zf1fivm*boITPHZ<0n=6vmJYf=^f~#>&5SAq8@;kYhRo z!%6}wYY2L)J@#4i(KV*}e1d3)KVt0I#^?{c_fAx;k_QLMnZSAFp^-LnSR0$yr9qf2$#{5MnpF@#L^{Zi>J7q@ z3y-&+RpE zw$C^?pZB`WfT17SVg6{hQIj)K+27 z@g-kE-poaVii!xvE2j{jR?s3ExpZ11XjgxZ+>mk&Kx zLW}EB0ieo)&BFODE9~W8GFc0EkSfz;`~bnW`p|G3?m2GFbFhJ|TNRW1x~6n*0SHVL zazK;70jYSGVa^uk^`(zprELmS*|hHgw=x!?3^y#Hk`YyEzNY%&-DX2)bo>VMxeGD0dY+TP!hTaqH52}S@88M+qA1ER++x~ zt=hOV(sg|JEl8|B5eq?}5)j|I8hcHV(uz3b9az+5H1(aQgx`1+Js#yVoh=WTF2XCm zepkHFEnl*F)Wj-?zbs@vZ>hh}nsXlnRT-)swpQ4mvNG9aqY45@4`uKaqa<~42EezHLt@!&r^sGA-lCqGM`ldfm*sRXZa9@WkZ)KU z8lH~h2&YwMVik(83YVcSvtOjSQx6pSGP(}Ak8??(yGL_NV%iS4NkzWd!&}Zz(}i3} zbC%_RagEmXV$uHSz){(+|9p=lI%OXTxoQ0j*&V3KO2yONAb9g{^!92kI6BkW8Zuau z`3wup|CRpUx4Hg2>&{GrMbN8ZCTg&LIMuuJ1w^4EbY&|RY`J?Z?Pf>lYUs*j7eKK~ zL+;Rv9c`Kh+a(pF?`~e#{nA%}Q#y?N)BGf^9Gd^&BA*l@zQfspAG(=y4x&P1-?1I` z5P2+;UZz*L$nN5i;r^(L_e@OT`goOPSdJdcTX6zj)l7Ocgowu&mhU}<+=7LQn@;$) zVPQbZ1VU*HZ9VDK5QE7skSy)0sR33T{B?{Q&sa%6qu_YS{gyV8e~&f*H%Ld-E(O1> zZ_H!%xZ+=bT3xHoK^N&YMOpD$*pO*f#jMu}JyPsK;b|ZY0}}zB35o}tWNQ3QT<^k7=gDA**t1%_1vpe}l5*Pb z-p)%tK+;79h_R_qVR_-3RhT-2zbvS`i9kRpcHQK>0V%k7{kI?|c5)Z&Gy?!(_u#4t zlO`}v{Y~{ckl0vP>?tw+D5IR$_98EB94T*AUm_<)&lL2hYZxJ?#w-2{*%tb>Vrx!( z{Vlm2Yd<+DUx8ucOS2AOxLnpH;UzY-zn`+L&*clrJS0Bg2n0^Kf)lFc>dL!_l_B1^ z;sb;r1g`lR`@bgoWLP8j%PtCIAo{G2S-W_x26WitvJLWCmT?vzz_Pz)n zi`I|+T)|c^eEy>hM@(Ez1|v*OtNBq@As-cs&A{1SKT$KPwX!+|TX9W)*9G07#9y3M z+n6RYMAPyFru<BteZ@g4%3NVk>^1k<#3Zz#NcJ=^jLINJbovR3_BM8G?lABoMaZZ z=r7#_!2=qzpETgv#V{DLJ1kAPbir)#ot0pPUFB3R zX_9&XuJ=59lmImbB*wRJZpP~k5ZF*3=0$etV1&U$eJx3Rpan4*;dS@3Zjxm0E8|fh z>7i<;Oq0L}Six|O7Ip8h_3>G{2ks4+ECvmlXC69?CL5$}VKD48ayr>J-Zpow9U^Rs zxMkogjJuv>LIYjhKKf9YVr~vx$)5;t>wJ8Ur^uEM_ngo+Uf*Rfw*v-5T|Gn=$$`{8 zS{$eW-)~rp{DEmX6XZ}a>zy_SZvHNFLgq70s?uM&QPL|=8Xs@-%v+AY9He-xuX>P` z-$^YU`m_{agy$fubR0jAMBQ;Ds5+*}{7O@cyXB8*c3pFIe^>Mgd ziX9pHp%?dz=~!2U$)T5;VfpA=;Dm38QLL6@oPQ2%cKigJx-DDhwW9Kw#Ns_b3SNez zriB^b#1;`iIj$!STFs$YE&ab3Hg$TVWH2ql5Qho|T$z}x#RrA}PdS2o&u76@B1NJ; zIErZL!;Kvid~oK<;=Aru=}mQZ>=s% zx|FvfyGMkQOl-e04SDTdl}P{OjrU_nuhH&c-fQ%YxE>y#ZD$vfJ*4Fp5@ zEuR9o^*%P#ojtpqT}mG$t~HPCB*k0Tvd1kAuO_VK?-PjMtr0NB5Y?-(5Bm?{uA?)iB3a6t zQqbtl(Rm=Lo8_3H*ID=1C}zSo*jWu?&$&OZ7kcTrDa8hk0>oi zl#sf?s;lR3!W)KOyxshkN8ft@fLd=EbR&YCZh5Rywo9Bd+a=Q#mD zCmTBouA@|*jregJM?5L@?vtoJY_l5)MwZmW<)mUPGsgXcasMT2rGzh!&KitX1Lui7 z&R@C(aajiMeU-4Nj!o_!V8;z7bb&7t(x}oVs5iYSMAM5QDFQ=X+MN1F0NX0U@79TlsA}{l=-}% ze*Ji#TLzE84zb)vurrH2?x9r=291jO^u78-E;|jDdH|4(i9X&*8t;V0B4j56=C|&r zwN%6PdvU~91N}rU0d<}{J?H>c2A1+h&b}tu%RSo#eYs(Qc-|lHyd_lP2)g%m#tyH* z3g@`u3X_nO)7H#nz3t1DC0`n0)q^ULMOb)-QQ*Ts%=C`}e9ebVsoRo&^^^>?aM~!g z4+Kke_vO@ZgBy7`J=(GZ2vtIo;|I{Rt71ylOWN7b~VY_d_6h{MhYX2SG!! zvHL-6_R>zW&>_y4mn4Yml=J{+tyE%p1Ybsk?AFON&eoT>P4X%3^lSop6EcZVho#^k zDqa(62F2Xx*teyvMz67HUF2FjX;R;ZdVOOJ2|GiGber{D>j%L%m35~_chk?!ASHx- zQ+P$hAFfmCP>0cW1X1}7JPWy+5+qTvDWsM102l}^;qm;zCjQ#42VwgG4oqfax15x# z9JQ88s|k+{5WBKAQLPZIP}XaK16?;ypj5(tD$^2rbjJcH8)sD<|8gIFfok)pS~rX4 ze({RF@~2ll+5+VJ0mpCRODaWXi4;1SRM2A5qg(U#eob2YC;D@GR5gZ2Wklg&1aZ$% z<>@|1Q`1l7U-dOdyZV&Tja zWw|I7?piaMC0$c+uk&@T5VFYKuQ{*iZ8Ccj0nSI*$ZsfMNfNrGpO;~k?!eFqNig-

QC6MHl-Ni#XX0jMZKr+v z{y6%qMB)`XHXrr&rM+?9>lr8MsTn!%AN0KUJ|Uc(*feXT%*_W?xASnksr6%Q0_(K| z|59J*GY3ya*7O*^dm~K_g&(*LFHe8r1q)W6hzf~XVf?$$ zis&(Q6UD@^%BXEzcb;gFWjqh#Z5EAHBLkmQpg;mBk9u>fA*!L`PMzVh ziJt>vkL5RZaHZ0$GH!+RIwW@=Hj zm$pAwWLx6I=cDAH2I}fHSR(sBE!?#lrkcQ)|@Js;$Y=QSXCj#tLbDn6hc;yReyfcK%NWP;xO zRLc~5s@+59+TCAQP0#5)>_aIzFS@1G!5XIPylAlTv(yq)n<$~R#)>ihlZaL$rt*i3 z|H0%%nEP~*wL5Pg|977{>{OiCX~@d+48a~Cs(kXXCDQC?RgFF1n%3vOQs|`1xAu;# z*%OfcQCiy|-F~bm9zY(JssUzlAWW>GK(Z3K%oASzh_I4`dNH~k^fi1PO%^?lh_``} z-Y<-wGV_WM5m8-an8def^26a_@%M?i1SYN(x_S_aS2n^;TO%xGGTPmq`PX}M+`KS$ zahNuYARN#qHT;n&RDoM@qB6TKXf6vgu+-2-VpWULbWzb00z0)SoA|Q9U;ms=pD1HO zHm+yTk$xxL=N5?Iz%8LYohuZ=fjM9gzd-@YX9Vs?FC6*tC$ey~A07PFkIq>L{Rp)m zDH6V>f7ZgbzX;b5>?#+0YL6({O)-5yEKSZdPrs5(aI%`g2@x~<7nL#2CK?oiI72NK z=G&Fd6PP>hkWDA6mG)Ykgt&w+fB>XNK*_${2WbjDxO^X{agc;_`&h}7DT*gjTbd6L z5(XR$SO>j!WUo3`tr^%x|3ChY+K!1@=~r!0~nOI@qkQvCv-n&FuKZ0UBvhi=BU->Su9bLYwA$c<+g z$6wxJJ*{kKF?-C|hleM*R_X*bULP5^XzAKmrRioxJ?`P1?$p-#MUkBsp5T~+y4$z9 zo=fS!GKyzV`C8~&Rqaf_efHA~K?PKwvs@)T-+#n8$a0G%6Qyh`|s>i zpGk@7pI^K;-4Nk-=;fZY3%8QEZl{yE+ZyD$+w4aran~zi(j%pa^U?}}a zY4YKcJ%tSSMGe~}B6dzG7Nd5`0!Qq-%UNepl-hSO_@TIEK$*8v_x%Zu-1YLf<+UoA zuw#Ndscw9A?SbkW3KY3Y))EmLM|Isi-7t&_vUybZr6YVDmS+T(=iVEev1{ED=E54< zCU&i)OP=_hc#6c{iQVA|2iOYkGBQ5J6Iib9!(cK12^aZU{mQaFN$p72gGkRS9( zU7J$JlfUzluv##$uUQk1+_)?Kz(~N~Zx1(eT_8W*+UD?R=1!jCIBOqDx9Ez5z^7PD z<;_w@iW;+JXID?f5l-*$$@^Is3Os8TUryrcx$8X?=b(Qu1MBRt2M>zVE$3-V=5Vvy zn=;HC2Xl{1)?xFo-RAUmhvinP!bIBdaz>*`AYR}t!F@B+I54*=W{C9FFUld2J{xwu zrj0o0S+6#gyteH(&vE$XY6-o>bKd-B!B6$w-74p9-z?{TTl1wHv+NAsa!wsAbE>vP z4%Y{_&1@LXK7-p5`~IgPo#C)MYpX`NjAcV&?HLl`YLmp^<@8 z@d1GSeql#nc}rMLJ{M(}H?c}7{s4QJR04xqH|8_N@%Un=rp>5EtfHx%p0uPbqo7S)6&_&i=;GR{wbA4% zleqXx7Z&Fkyp|3N(?3~yb$GyKcN=wuTI_O~7n3B%G+#AQyHPGbPW+bS8eTS8cm?wRci`R4I zumpMboJCE{oBYsf!>_gqF^@|#I)UYfB*Wjxv+5=glkY@6x&T1@7sDdzu!NH*FVrxee^jUb$-DqK7YY z2bgN*NKrSUhe-3a;eYg9{nq^EPWCOLCwF5&iFY5rr*?+m(%J5n7W~WYm6f?9#GgR8 zO}OX!x6gwDZ=ZoHomm{@kbiPvI3~Ij8CWm%o3Hsq^PaUp*VV;yCh(1$agK zc!o)crCPg1P4NUOI(cdm2ZBD60gX;_RmW!Fr2M((W(Z#0tvWA(MgI5^i>$8fnVTy% za%V>7zjG_a6zx7uN9OrZ+Wmrq%}$K!$yJ8N(TW&VLA9@QsKS8B^ow1ol@K(&aB?di zEqb>(cY8c}8TMX-g|Rfjn?Z8aX(L% zbQdq@BU#lpI#4KurZzG?K3^5C?(~IiU)rtRwbat*eF5jxJhgMPJ&KYHOX&poh?N~3 zcj{#n>l+XqWBpz$ah>q`SBR|z+4g+mrAZ@FeZ-RaIoe#_ZEv?dn>CtblXcjrY?;q* z8^)<0lVeU_S|`o7z$WG1Mpn|qx~XL4*L}&YywP|sdEY4o@R^NHXRoJ=>oq;>HjJv> z=7$qFQHX3I`*j2RrG!2E$M!LeD%vQkiRfkCevIcJ0w$^LNY)f#-6_j+S47J#K_mryjpy z=p20mC?WqKZsv7HfmIc<10K3jm_$$iCCc1Rc*`j=pQVv^2F`FckK3xmO;O|6?y!ki zVnZGd2gyWemGHL5Jm~Nm38)~7A&qQd3@JxQNbnWR*Ou^i2v(pj#J-ad#P#J4n@MbI zG!=agqR)xCI(sZ&(#s-xjAHub?jv3AOj~)F6CuevP9<<+)my|e2A(3~UnOp3k|&A6 z=l?Xe+>0OABli6n{h&0oVVL?vVwSmz%()#}+LXrHXVQE(vN103SwH?WjL@4~aK5xJ z6-CSq_ey?<9#a;hQL2rUWHo`923g=%z$Gg+?SUGra4iQ5`o)cF_(f)W?87gNkTrK@29|jTzJljM zBC7~XFuwwenx{Q*7>S>8h|w3Wy_aylcat{8Cqf^re=5wFRudXBOtg;0M3@?Fat6aAUZ2>SA_M?qqilK`h6D-8iO9Gg%_F+xN zMJn`$)y$*Agq$!;Tn8gdhl!(rEq~%*8Dv8iF`7RNblFapX>P%xhlO^d8jv3`S z0sUmB1{`UyqN=e<-o0|&_gb1W!SF$g@hvxRAsbUoe75QYn8X97^$cbijZU6l66Pf3 zjkX!~B2SGo5qherqm*E}`(*68NmFVOQvg zP4*cb*NSPg_@0cX7s8R2F6xFOwR#i@G3Zs2%(e311f#Lnld#uIlDJdPk^4BgxE_@E z+g_L*SS%KC5U-nQOT2shTi?bG5?i<6!HSE7@X5d2JH59Nbl;21Fpb%CzMn8&xi`N; z$2nMGz%sLXMz<#X=Q;78`aGc#`dt%N6Xy3ws`1sz$rbY;rugrfPD(m?Ul@5IzytC^ zlcT?ehJP&hNAxrwOED$MogN_!>6{=YrN4-2$5Y(9!-uu{_YMNGnzlF&@;g8<#<-D2 zdFg!*u{H$QF5c;pv?@{40$i`9UOeZZpKeIxloNR27tLDuUmScc7MLR_=AMKA<){KH zeq%#&#daGiBCTJPn0)qLh1iT^;a?DbHN!aMrwme(>{yb@ieZ~S>GRCXsqf7FXEX}fCwGi94VDm$A#JB zIRO^?SKR4Q6vp9#C5RI~7EQ@0hgPbFx#S}(O~vz0{v0KJRbnTB<^)_ z1kkf}`?L&zo7Z`lX_+dq^ErJ$$Q}i z+qkA>p!amDD_tD#a1BvlooOd!?-s|wDh62YlHDsly%j#f5Bb&Xd6C>hIp!n+4t9}1 zqg<6&Jl}j5G4It_MCpP*;dAU}q>l@Kb%n0gh!anrw_j5CT8TF=`8~k1EO5^Bf&>%h z#5_bB6Te7Bk}`F2a`jiyac;BFTWux*6A|1z{Uv<+V*X!~^G2ak$S$4`G0;GVZTJ{* zR`vtQ9M|!KC#W#akkF82m$7bSGS|>9t@JRB9?UPm1*_>CtNLjbY-zzRX?!ebjtHHf z!Po<-!Q=MoEq+|uFg*5(9tm5B_zzkr49pan5PKGOYC>e)v8soCozsOmO54vwB9&DVJI#1kl!L7kXW z+1lNerz*@ll@otP;ejwmL(DPLu5%sxX{20L(=aaf3bE|mWo|4lWNJ#qu%_7B1gQ`^@b~ z3Bibv%Sq~CDsRRUePHI|iPn@by4NABXOXae*7|EvFAR4+Q-|4QNb^#piNV=s;K<{= zM*v}`Dzh7?*buUEC$Vx8l$m*vreL4@!I?=dtxwLui^=k}#^fbyR%g0=Nqi>E+Z{@E zL_^8kBt+bzO#s@B4U!Pn#wMDITrt+-^?jWwyoJJubnoa7aQjpxv4C8X&+M4&f;CeB zKF#l4DrgDKTvxi!zIwbiCe})L4sHkm`)WSu{V5ctFz54Rb?9`n8tEp~kgz!BQv>4} z`sK1MbUhE2hcdDMl-|{IIy%PAQHsz4TLz()-@p$d?9VWf+xV-Mq(}tDFVqjk50zv9 z^^u*hTG#SDeTp~PW^_!p&&Yl^L-2_+b8kih^V>H0I-dS$Fus`1woF3!mHAlp| z4f81$GapU%O2L^Gb)4W*2d8aG9FCVAC{6En?dj~`8G_2QT{934ajqYelnDnn@TdPX zr*$}aqrXHeeF7(mc_Rdh-UKo7=|?e~1ifG`AABEurTl1Q4h}JQxTPf7yihD{D|T&r z3nfj-;;;--|C%#Jc$j{+A-VYp%}h3fN?c9&DJ9!mC3F=dwcH_7&Rs2XOsY8NB6zV+ zDIdOjL(vkFaH@ zo-vp!X%+^FfmBXX%MT`sr+>?wxX&MWy4S*mMxn%RT!|(K@zOgLWRox8K9<75cvkv6 zOi`{la@YPhs0X3<6X$DZeUN& zBG@Kx)gTrq!F8;eTH=6d;1a`6(ue0bWL1Z z|Cmj&*v`AR5v`;X7P4p4H^K1<#C)T1avfdH(HEz&cm7VSm(_W)6U#L*IRaEetZA+z zZNV2_$aU}B@R_ENM~49dzEa#F!P?Obmnh8rjy9w9`d{~?uVuo4eDiPDE&d5>s<63o z-TbsJzMMOb1WE+b(?15Rqw$8opVWroanp(OUcKHqNbh7|ZlYsYnqW0@0%&;^!~d7c zDLy%|%GPo47=nV#X-m9G*J7v%2~8+_rMx)0 z=pB!#x2~bcPlv)MZ;KlrC$%rZJuNg!3rIYtK%LECFsV2PQ3BcjHG!he6S{I~zMlobSVpuVt5z)fF)^ z!pDA1mKvON+)iYhNW@v{o`MiST@u?6zAM9Y8jVZ<4TOuS0l@VNdqiy_1K_gwJ{K)keoE~x$sAtu$pfUnr&g(8XZzWyOY z>i_R=?39J9BdE202cL*j&J9Z;|bw5%1P$LW4RkF}&!^S8kg z9>x+9O)8w0JzFPFN`GP_d5EcXV!edRR9{bHtGh0{g?}5%`Y%dF1pWc10}{w`U%f&? zAM=;mXv?A2WNvooIVD{K&R8hcPnot(_a~L}K@>(=#jyVq#Qx2v3D+#{FKmxy{5JNm zUQ4WOpOfQY$}Edm{7D7pF)tX*fI?xJ&RRR?XZ|G4^%nprlxKzA@>$>GgtDh=7=QqW0Le7LsAgJ$eXeUQeaO*w4`x5Ir%; z4CbQN;g&!FzjD2O|8B|2lLXfKKH5|5 zVW=u3E><8fOHj|Pu(L?`{LTN}^*s9l{AEBqw)NYEBkCrHgn9M35ZnU$ak$x5ytt@h zZvdt|dY*a~(1(poC+7D8aCyuQkb*fEv9`PMKPZ{dD77R!7YcT%VKMzkkXv5IdO>4r zWQn4Jx-oGgm!BvQiDTf@{x`yVu^)!Cm>5#anjA(qskO zHWCzTt=SI^ONMF67Rj_xs{(NSmH5{Ne$OX`Sl{TA35>77VdQQ|Hl%tP~a$T_VfqJhid-Wx={q0|K3=*@NFb7X z#^eB=pv>Xk`cJ?(JqP=}5Bu#-+X)}6h0ar$)ijZhHGOquqLRoT%pu04(ntr<9d&8$ z9qQT){b&6cj+_^S`2-<;sM_NPUzV%G!@Z#h>Lh!{Ka_*{- zOj=%a24b!`hR6Gh)E`T-VK)9S28VS>tq;iSU5M?ECK)aLdirt0QHi13hgw+U5pLzM z4z?CfsA*}0$)KJ68;pztZ#(y*0hMI;r$OIRTDJt&k#_di#M!qXke0255`*qPjaJDe zkG`_#dz+CRxu(3)$Z=5cKZyG$_Wuia1$xDc7BZMQpv5TGvFNX?hJ>7Tzns<&Ub6op zy#s^Cw~ObiRFHT9@WY)6;?yft6f<`!3z5WO1Ly&0{|F)>_z?{3-iZ-PTH=+8teWY-g+mp_NvV9XW-t}?>bIjBNl;ut@N(sCsUc(!)D^bu@_73a5UK9-IPTrYZAzxucX{vgiWn zxL^mVNzpOn9AftyimlDH(niGQ&z1l zZx~0KDvVu7WN1jxRv>&_G?^1XTK9r%eDWo-tGOJC4b}q`8l;JM3IQZd)8!|EGp7{W zaP<#qT$zox*)IvCRvH_(FCt8h#5o(bf= zs9!+r;gb9Rt9KZNSY+}LduSzhm9$?xUeS1IFzeqR;B3ULek|w5M^elKab!u!eaWk~ zl2_g2U+n4CcOUOK7@5a5pxC?b%5QpLoEO_?1cKeAnN9kBcfZ~*9pVrIRjD9SGn3_M z5_!tU4|qA@dVakLy=m8C4y(a$W#A(vV-v;uD!b5@TsXtUf=@^$s)dA$_(UgZxFFm1 z_b-$e&B7m6CfjN-Qe|fUNb@~j z?PH?L5wR0Nlx^$6vMC~1E_#mWuKT`1TjcIw9}+7X5i9P^kmko;AZ|Zeq|A9?j&IaR z>|P0|wf2X)6Y7B<#W_vy(MFoCT!BK&T)S9|o3zZ6rS}3`KaL6U1Ml_`7!l??}B^-SEv#NdaoLzF$hzc=Dgkz)kcj z0-bQ@`FXH-U6WusB}f@2Q$)82xFix4786FAX7!(x0P-cTf*gSazg>vm`XnKMRXu#| z`qz^5$4=lT694Fho43qMxNwRGNpo3YfYm=g;N5&JVJWu<<4m6n+bTuY&~j9 zr-+(j5qDrI|GVjTo@B!9HW3>UT5h8EfAh64a>v4Usq;Tz-u#1WJ9@1}sW-tDsd8&0 zV9KuFd%il-*7@rOXuLr!LyBu4`+VnL8-Hdtu#Y#;Uj>`Jf?K6y)mDgHWBa>rS{td@ zp&|gho@^gk2CEIw0OxW@Mk#< zEtim8di~$6vVCNE|9p|F|B8O`7e6X#tagBQ0WehtFr_6+Y~|nkC{D+p2>*l;av+LQ z@cR;G+wyzKp=40#a#+qQ?xTu5zpU3 zuaEys--PcQxsd4h2E{AOVAo?hGN7Ag&=n)f{__RzaYS(JO`dz#A0jhR=f;S0{4js~ zb~3s--S>_C|1_p4NA}%Zy!aKwPW6DrOm;^q>MnZ0b1eFI^d#=B?^pS0kT8D*4yzsj zjV8gogyy7H?D+xn`se?QdB1A)iT3Ooga#~STU3F&G{Bmm_hle7*M60*`sVMg4D~k% zQ!c#rG_ksrtV9?sJ{_jMbekdxrQ$p4WtBq8O@Ot04c?fotRbp(KB3l8(Ppq`4EqXt zkwgxZ@PM@M;k%VQU9=*`CX4GqKSX!Li2Ox#s{)~KJ~Twgv=cTnLM8fX=%|G2q`@yu z_C7q47ocsx@w(E#SFnma=oKG?ni0}`{&f-o6E^gNysXiL$Zd{&ZX1T22n!HIWDA?xRXhkBizfq;>%c! zi6T~3-4cnwek)YVbD_>g-S|$}#Fs}ehgIbcm^pcu5Kc2`I6=S14O{Zvm&2zoeZ9&0 zPYKXT6pwN9>+Srf2ZB?mE7lW(XvIshAI=5J=8=HYbxq{W`@ZX66s%_T>oV*?h_-K~pxgXQ!D}VtYyX$5eZj3U2UO{5aJ1DpX})?1dFjhPKwZ`oXaqGA z=CJ+oq=I+EhX|7QFDRq~qQ)bokV_J?`(s^7a^5fG52ax?QAy-ggg&e!!ES+QVW#_t z=u|hn3ia_i0iQOamtvh+_f*EQ9kM!&6(`<$6B$9OoY9^y`tV<+;+{q868l;Aml#l-?d+~=pmq!x^aL9 zZSV%f;58nhG-fXw^QR7HzI&oV$2_U{BVztw?@wJu7Yt%>zRQ%P5}XEq&XZa%IioMK^DGYZm%;8o!mAhl4_=2=1h`CDZ8oc3_HqdDgxnwf|_R87?_K&rX$WSEs5cb}Dw{s8F&DVOQ zOpN=%JN0Kk@KzyNdTdLm&lW*|4`%f;WX=}F6Ik&pwRHM=TSP@G>%wlKpiMJ0lGJo) z!$NZm3BYl+!fc`twhhDP^_&kpc?7mfEfw>1EMy$$snPjTP22o!wf!m#+d~q zGJw2q(5`ROB?=M4g$90qZ${v?-|4P<>?u;JV-lw&9^u~u&bKot8@^#(@*S0$;MgzX zM%2AkbT@?IkA8!73)zdyc!j-xVmLmONX9K-r)(F2sKnY-}R-L5V~?Uq$<0 zXidX+$v3qlrhmFxiJr$NXXhKGTjD3)u`IrfNXBMJ)LRmWn-p^Plt#CA+OikqhU^*CHPDGVaGl zVT8js!3lOJwor)2=nSxmE`@>=21Jwk+!Xt&ktr(HR-g}V|E$#8W-2U{QhBtDBtH}d z`=D8(f?f(Mf7Y>lq0Hq3UmcXOYCr7JkQ?8hOIM{vcXAr3XX^sQxlmC@z zp%U*tm&Vt8#CFC)Buur5Iz8;&BG5-ghn^8;5JNh&ysj37mJc^*wsPJ~koOP+OPJ3R z`l&XwNv;KZGcq^P3#P5wC5$PHJC*Np50Cv>bIpjo_Mf6_D#FaHN30hwHRcnseNR!s ziH!zO5;+M)?cQ;f&dxSO#x5aT=W5Jh?O*Y;`h>mvMt>hAkn)G-0yZqRmEq38y~dEa zH4`OTf3k7q8mjPYyEfZz&-XzmDq^dyL8sR%i1~tAT+4!O9?&bl?59AX zbRFS(koGA3Fu6pXviqRK?IJCmoy5ysL}tYu2`$-jgg(|mMegl!;3i)6z7x`cR#Wje zM8d1Wx%kK5W0 z0J8Rd2Y?hyLy@aT8fDcOaH8a{jS+0V-*x0PjW9W;M%nlJhqA*bEQz?8vZ}*y z88kd+-(wrqH5ZohIuyIxeMF5XDtY#9O!sy4n;xQ;*NAz(Ly=5hb4?BZ41%N^N~9cY zz3`IPnQUq8TxdSYgi@Ii>A?@POGlz7ObL61kg>$r8kvjzKH3m*9)%xUMUg9kM$o@I zGZq&AGR(km81=v8$q~cAv8iMWi&Ob0jP94v8dy!jJQnrQrpGDWYsA{3m$^8Fpk14C z+W%eWMX_Hyv(k&hX|l|bM!$SpC?)rMYL z1Iig;&JfNbQ>>#c_+#>1WS+^&QH0Oc*NIpQN)r3IY%S}pJCr|vmxN1CU^eASD;n(pK7m&v?SDv{s5T-(q`X#XRi~8 z8OQnNLpY-FvEud*MjW@KAH1U^tHnIyVV1G}WW^r$cSfsA|3meK1^GTSUHu$j)Ra4tqlXnl{>4q^+(zGJ&M*?#pnn2Om7p@hRM3v(CSFiifgpO4NKwF2hSVbbz}0DEL^=PgmW@ZV;}5^NMaR+?}602A)VeIcleTTQ|*CV2-s?C6j9 zmi&nL+$~5cgM=ra&FF4K*tYsWCM19VwI|2=vlguwZMD8vQ#*sJWP8R()sj5J+6ch< z?FuunU=XN5Xb~G^PryK~l-LcspS(K}CNENqQNfbZ{NxUVOVt|D=FqKG0s&2?5JnH> z}WWettVG2~@|Jj<0_(=3SFjtt5%>jV7Gc5*-n8>>;XOZK_qHI(}a(uDqtNk>{ zNBR%+y@zg{e%L%%c^l1kSbgZeBrMI$fA7cO>4&cU*0*LilisSj8tzN4OdYmoXS6Xn zI=`edH`$o$;*%{< zDkbvMmJAjMUaIB2Sno-pnPCRc?9Ep*8}QpNBwxh#|LW%%U-wiLy!G<+^%^VF=Y}WF z!P7P)=Nrt&C=T8-Jdl1%%fI`+9JvqdOC+rNufF&BrWQR5PH$i9>>s78hyv;BMjmiL zcYxLp#>J-*sR*XW3X3#wN`SGY`{++(9t8SdM*ct{J%VYL_ol6>`=V;QW_-DWO2n8B zFH>KA4oc76#@yANHzt3PBDbanPo~=F0%C>;`eWK|r+4Yznz@X|tyI8w-L&yO;zrf& zPi6&vo?ePeF`98AaHDCa36szw;#Y@S4NJJg)R+$K6vkeM5rX{Sv}gp4Ep5o!b`b?!Kv-f_fNFoBsh2djo(laLeSN<8l%V^MaQ}QeT?50ZNxVw(A?Y|@iW>!LlBSO zDj3*J!B}ztXKZ>UK0u4Jk!%pVG2*Ttp6#qlcM@cnY_(kY>G0l~;m3X;S}C)o-|AlD zr9en!-0EzBCiaHxk74m8^B5oGrCiiM!N(*7Ph8|=>=PE^t%wI8kDH(E2h7c$f1j}^ z@FPRYQ=ct$WDQC4^~GRJdWU@aYLcRfxKeUIGyx;qx=oM9hAoV$t$f}O4w(HzGqc7r zLbWAA=e75cT{|e`roh8%o73dA% zH2#=UEN0D}m3E0%y3a8Q6LL`Mzu`#^!mk5Ks@DgfmbupBh*Gwp%_QR0*G1U)Ml3CD z(@{%*b0T^%dKy2GL_t+r+H3_EfBUmoD@qQSyb2Os%A+p{8OjJ#3GuycP8=Umx{1Bp ze^|-%Z(;(z&KEQ3<$N0i0Ozg3;X!7(gkxQ~+OLUT*pk5r4~rUW@Y&K70m}W_G~$Nnzm#0m9Hyww2u4Q#~&~``=VgtD1Vh>gSwl^ zDI|e7z9u`DD!9!sqps`S4cUgD$Rp7dNc#7RP z$p{cEW&$&f5rBm8`|mP+ z^O#wj^?)5X6~_Rw29)NE?tj~iJ@j7WiZP@x_S(4JhqW{&?PBlhK2_yN>?kQhsB5CO zuI}BU+Z%YP@;fs-bkqEbZXDb2S*aCb@r%48dW-}(c@w(CvN_wj%Sm!uLMj$pXI9>4 zOw3&$6;->-A9@6fI~A{mG9BvLx^J9#p_x~~#>=fuc)kh(cgd_29?pFL3XA$1$R>GM z`59}!k>%_ezl?E)J-~@f`Ra<*QBc;ilyN#1m0T&Nn@66@^Q(zji}*RJ3?&Ps{Klcs z#O_oN{H#fI#|FfzWb061l_Appv4LOs?*GdB5`QS$@Bdqpr#whWi=wos2t`>#g;r5| zEJ>ymT7+b0W=cg`C@m;MCDMjeBx5M0MW`&(0Ys?xMa-9Hs zgVO@)9*(x$23nv7mP>bAS>B?={!MV!03-yJlmsUsq6fnlB(CZSA!0D@+EX-uCJAp% zJ65Ca-^0-6ih8jTVbkrP7HIw8{q-JAQ9K7taIAw?**30j5>3_^+Z7u73~uIhvB#Zr zxUjXrE-1k$2a#|HC|2&`-F}@g)=wjdjIEzETE*+^go;$Q-eTz*utN}_OcA7$oGhg( zo2xABGfj*{WvW4~uqRfKVWE+K&nx3gQraCNRRb-LIQ9_wxnsIQl<@vdaEQ`y1(%QS z8|5Qh8L6#wMKxM+|7w5Gab!wqeHL*QLNT6^Rn?$XS#yh|w7x*gzWuaTcPy0lHyf)I z^|<8bW;|Wb)41?!X)AuMrgJot5RO!GEnoVwzE$jZ2sqQzu(0!u^hTf&-&`Mr!8|5p=(D`efs4^kRebDZI_9TOuZh2P1D z-G{`EJS@y`(iSJ*Jh%c4UShTE2x93z;m-a`0(8> zTTu#OkkE62>^_vUm1JXb*)7w>nc%nNeCGGzL;jacwS}8V==WjLu9=Vo!3u^t8vovdOG$;uJT_g- z4m=1`sY)S$zCa*60NZnYCm1abhyA1G;;j}=&!!<15A5O9(O!$ARp`|9(g2w2NgDD& z;1|A}grG}Im%hhYyLCesUU%?s>G-&dK$483V6ACDl}^%c%sLpNwUFOistH(XkCh@a z77XP9v9*VI{^V@nck+!o85JlSw5GsCbP^)9@H~npyR05>gbbksp&4WAzgMGeMW8jv z6>-7j5K=-p+>o~rGXuN^gc~V@Ti%c=E}+Yz*`;F(HL-_@e{IJwnGHunzJb#JZEyK~ z!bILhYDdfG@ywE_`OwN^nfR?}$arXqZ=y4DaM*`f5zt1k3Z!L$!39vf$KS!>^KX5< z`G9%^@t%#(rpDGYNc)G0ATWeX^JgNzsS?Vq6P`nLRECuYS+N+fu-0l$E`RISE?pY7 zag2D=RUb*pQ1$?)^D-(4*M^@g0O4o|exv_2pF)t1pv#Uv7->MnrtvZzke2Y@`&$)e zZk75Y2&-5TklGMFY(>?%fSm<>{z7ixq)FkBc9wlMpvK+-MTe^_jHU1#J>@ulln8*Q z4OqDsrIVG}$OR7qsk(1WfY>;S*OLgdtTyw!!>zd4!aGF-UQ!NOsJAErfS;n0AE_8T z16j?<-HE-oXh6NAMqp}uhMBw2pdW$&OA~e@xn$)@c04crE^=e^4u24Pf8fIL>t;@Y zt+Fac@oJt!;u~ViAX!u7MB9ZSu;VpTYR!tgvUZ(hxw|}rB>U%>h;fw9xS$S#sz8w) zK@w_yr|iHfaB_$On}7`|2ucyyP^U*$d?9}6z%#-PrFl^(O)}h99F6of?!{noRl|gb zUZC5xfSsZAfClab#m;}>`-$?&{>rE!GKxeg`vcf^|cA6fFR=PlpERKPq1>QgII* zRV(D1qy0h9T(#`LzV*b7%ib`@_*Jk!K!1XuKOUHwSo>;p6AWx!8z?zM4TcN?>b-(u zyw@%1%F*L;NEuB4gEdtB_6fRfF#9L)Bno6pKI5``q5H`6S_NyCO5GSR8$QCh7usdO zT2G@G&JEss6zVd3a`2iLX>|Itv(!{77A~eBpX8D-qb1#C@vzn53N4h-1tO0+grc000Mqg$Td#tQc)o`l!)y<) zt@()kP(1QU!oz-KJ?X zzR8)7e5^Isps?sn>=vLJuW#HVB7)!kEHbbhr7F#e5A1VrduIi=Ab9BdWJEPj#DJz8 z)ax#5psfM^0WH3$##PFn8=Tcz;%hh5`o~8eitD28a%_2Na|;yu!anYp zbme5UX{z#&;TB+_(M@!&;STzlcnB^b;8sn%zvT-w8^Ke+J}9iHrl03f;%j|iED{_% z`ac{C`7em;?Cs!ALxi-GUIRUnpWrqE{?stO6Ch-dc0Ugxzy(jSbq8TD0jLaOaDPn_ z`*RH+osJ(1X~hRX(EmpRG;h&AK+{^dO+oA42cT1{;mtS3y9ni`Bf;6cH4s7rXBUdl z%*939|Mi*=V~2CvG*^!{Mq=fqe>wt5PGBC$Rz@?IH=m+cgHvG>es6*v~pof7vvPCxqe{Rz>X`Y{sf-p##r@(A1DD|6gKGTB{}g7=SY%Cy=;-46kywP47cP?SNf=h3}pJM(6)J_!Js(v^>7WVA^g)|}Nv zmRt=@xePV#;4Lm{^-oEn>kP(Tq}P(DmjVJF^?7HNa#Q-&afsb=&p6$M*3BfhZeP78 z-Q4beYs+q#d&S(f)F5rizA{^Ge~cIh-K;#g{Lf$uo_bmzbNN(0#5=oM)%4b>YPpm=5>K^=$y4)oDY>*uWp3}v2^s-0=mtvu=X6|DecM$4gwjX9i zSXyu^n2~z_xq^9n2~sf_bPx~KO+ix^;`OLRtq&%^20Lm%aNPXRrdf>gZvEC`V&6t# zqWQ5-QfbV7{(TMTOfV!rC!$dH=B(Kz>Ug0&_~j#Y9~Aw5b!3yl z(zZ)ghJ6MT0JJRBBAt8QpSxtpg@9&aKsu6yabJeeUHiKCgb)fD*j>eq&o4GwFRS<{ zWgg;YvY(Ggfqn`f(&QO$!9@fBJbBc5v;e;nQV`nZ~3*7 z)Gi^==1?J6@ZND%YQ!~&PQUy&m zStY#zq9pUjfdSr;G`1pF1LthTQO-6GAJ5tSg|WxD_a9aF^&4^bayt>L@8$NhXs^7u zqRn1|)<60ilJ3%6duZ-~EJp~&5OYN~Bmu17E17B#0aEy%a-k&DydF_5al&QweMsX`pwb}DY;}9Fh$E9d>u&dA{0KEH_V*l zor)xgP83kUIf7(`S6YO+0+zo&x1Q!=f47w+P;d z?;zojRLzBdMB8(U?A` zASUPtntpjfg8mL6~&jguiLT+ zU5vHB)DzLE0Wqu3vb`qv5AaNiCzRe`#>(LxavA(uF7AnpzSS+6#pf#A+YXI&Jb*$` zZo?^efa-7rfruT=vGLiYz9DZTs;mkOfB=Gw=9g*x$GD=KGJ=ml?dfQ4Wq`Tb*oMk& zb{I(J){=6s@L@_VzquSo5;vAxW2l#f8oNmj(0kh)(g_$$|8na=Y|}o9Y;Xa_r=KOI zPj-^bEttKNpK=aLx~NlQ^OUJQ>|86{*s7w<=R*a2kb}n?bnNKp;))w6r#neiT7&|N zbLhTeFpK#qoAKg2dkT9OSE!&$rL9 z24<3fcR!`{F`s^qiCWY*9Oa*dV^FYeKE%CFS z2P|Jbp_Upmz65&56h`mR)^$LuW}fdsn7oHd$MCjyT|duD@pIw*??KYb%WxKHZuWz_ zpfIkR6*CSWsp_P7ssnc<$Y^~i)-7!Aey*Ib5%DTuU)t8&N0Dl~v_ha0k;D~Dd=!wv z>b=PQ9dASJj!#OObg7d;>IE;6k6hiwY1FamQs6cp*3;Wl$cWdr>X-g+RxyLrSKJqA zR>)Xf)W@K4Ul#V0qKYAr#wvVB`dANR5zV*=e7?SA`t^9o(=Xd+HTmOj4H?SjEN9ZZ zvjX4Ab|u7ry>&WmSrMO5y1h!~nmeOn&gv!|58ta14=~5Y2eTVJW1QdNO0fM=U;rR& zIg2=1VRZvMi342v_h8(*iZS&26Nd&XmEWe=#Yo%AT{yu z#Va5{Oe(bO08z&IcpgkpAOih33@cuWv7&0s`N|75v*$;_MPb4&ZIR$OUvOW$&^EO0tikHvrk1X zr>J4~;}71^}oYJCRv3L*vYaAs&zT&gi2XAt$TVK#u?-KQ+Z`(9Ghl>Fd zR{pi-#svj_u`PuS*&d?p1>bpc<9ftmAn=L9`D(T5$q?~JeXDXbI} z#WA-lOWYV10Vu@_YfXhP3}$LLYO0_-IS;o5I~ps7Klzf#KTzK?e=2NGv@oN1Tt#5U zXB?~JA-L(L1D9hP{EdnjPWaeSB>dW-+ynYpOk2=lY-?A8pq0Nl7!C4z<843F|3dAvU!(7%>7L9|;*CihFfD$s%Zl@Pm(#V zZl+{sPQM}-M*w{8#qEkz@VdqAUIRminwVyh&jy1Y1vWEv29I^Hf`}D`TvlJPDU*BG zTJaH!VWv9hr^|H;_GGJqrS{n4{WZ;_VD;eD3*cS+Z9)bI)jm`5^9&T`ays4eq_ zKmwjC8STZwo@nrVGz+B4qU@zrA))gwt)7o^5O@|swMHfC5+C8{*9=wR1k_mvW77b% z4m0;0*E$J}>X;7acTAqdlRK(^369NS3xdU^<43PvX2 zhncb)c`_Xx{9cYv-CZ>~=QsBl5i9&R_Xcq^5Aby3Wwxcr0FEG@gp5VqZ~Flj8-<2i z7D~fgoRRVs>o9SC=GsR!9j4|#2b*GWQ@!=+~+m)>>MVG9%&~yH=AGUJ&e`Tx$>BMSPpz2+u@`z`L^)!%y^iF6%_ZX zvLDA|IVYi&{GHLdsynh{Qi9gYa&H#ztM@I3n;8j`V)cUGH#&BKSqoEkqCXn3$k z{VV+93hC%)(ZPiDbPI+|U+e{`&4Zj5(G~7D$9J%x9+2TKvOEWL;0r5spzd*fo3+7; zn}QzmVrlW|v%9^{Cimw?>=@hKiIFWQw0EL0*5MEZgPB|=&5?6K81s|G5f2;|3=;&9 z&E4F@e&>Q3odhttRv?=X4zdYVV-VJjfS2SsAlrq|7)Zwtb6zuwjG7s;b8wa$Z7;(z zE{)hK!?QnlJ0QdbGI-BE<^IE6-stGaig*APFK!`Kp5SGTV^G?R{GiumW)I!5nve3K zIg-RRdCMqm({vddp>o?$}|Voflk&K@^CN-i1R@$vGQezC}ZWHK?k4!Xi&)pz&5t4t>_S= zw`qPEwjVN1{x>G_P}eOM1v3Lgm+JHfU}= z;?+$pR3nISp1|*is?OUoU%N#gj?-w|a^!e`Z%tleGcpMTy!*~5fxIn)>#Lv`0CW0E z?-z&G^nX;f;5pgLLva=uHGq#8_<+wT)YBU{#PrpG9{x+Gakkk$(ua!aKHoJS{&H?T z;+u|GxVYIRi?+c?uVoD|T2H4nJ>>mReG%C!0Y@R3ivYzkrg?+F&!P9X!r|eaK}pK_ zJ9wZ$=PASMS?l~)SoFNheX&=pA#7#7A%1c7T`H>#AlHqhC-zojP5&$Us{9#+5NCBF z?z=)|m3WW^AFdvGt;#s_ z>b2l_SAs`Y0?g>z?|@e^OVVMOY@c_J6h0|80PpBl5d#CLxFNdG5P)3?Q^(M9FIB_QHW(qODfv1HeqFGTNy82IqGJPg|VN6>4h;K7(nRMt{pn&A7RB@t1^ zfA5f+g5qk+fdeMPnH7_`aI@~B7OHFqCYnGPQ|f@7<6*ddJq%MdpL(DK z?6E=rXk?!pZOiMScLFzTc{Qe}A!;t~`e_6FAr=)whiZAS6QShm`DZv_`Qkq{S^o49 zO*S;_P8wxUj5o#CH2oTIb_@Qm9_YA%*NoHYjT$`*hKz$Ki*d!3e}GboiB&mIC49_t z?q(BjE`ZDw7B{YbA?m=`a>~4H2qt63GfC-S)E;6|wb5d;Csr$vZ2^*YV$T~LtvNV{ z?rjKcw&nRBh3HXt!PMhnPcrfNCcNMSydWnG7-ao6JIH8m^EQW-1pL)%S=)0Zp+ z`J5mLEip68kCdc%68J?`Zc$kqfr{qW#%B-{ z!iOudS9u^yHYbeAg20ntsNd>wHAxBgCfqR?%N|rGz8;hZt-8-H43*3&$@b7CA+Vb< zG3BeOYyMTZw$rbw1I(-yy1IPUr{2oI?K3 z)W6&#hKw(o);P(ViDKp%XYeruBdF$pw!dox19IRf3a=}jp|WNHAPPZysKE&%-1$p} zpXiFjUCL?rat(Z$Z#cL};Kh`PuIN(DX*bbpDe<;m1*di5z?jNAY)8!ylu zhre2P)`p-yVDgrRj2m7w>=%sAOnkgHz>>f7VBS7Z?$L;cXpKWSIP6&kq%JR@r1@6x zk>KtH1I|sxow1CHC5;SyWaFSY=ITB4*GS{6`!fQB<@|!}&$PSh>6)rwLnvjAVm1O(Pk=Fyi=oj5 zKn3U9>-baM4d6aFZ61@Q$3g{hzlYe3e7bqfc|h8r7OFB#3LuyLnVSbE;txB`ED!9S zDCbe#vXKfbo8mVzUKtExS>6lI{X8_KuElj<#2uYnk&T1!wK;fM+>~Sn1Y}T=6GnLU zcXR`@Jn&%11|BpP=#R5#%0y`HM1_D!-2ET9@p!yQs-uPKatv6~_QfAcfqZT>m~6cY z3^ek=5^p0AbapgNXDAKmBgB5Ly$6R4vDI@zBGxI3l)ljDNBbP zA#=dogrF`_Uu}`_jwU%R&{HL_PENceT@XZ|{ zp+RWV<;&5w+&E3Yv@YJ`6Md34zOYp{kk`OH3W>u*T(#yKbZd?;Q77+BVCB9=a&l}n zD;69#gl7Q3tMef4bv3-1^5=jS2MRN8fvyK6V+@O)Hwu5#dmBnfTi+p6pZN2rKIu{M z0cOO~L_Sfa)5E540d#RqvlUA3MHlc0kr#D4Q+O9v{bEeZB6hUuk156M^LX1Cs(Qt4 zwi!t=EY`t!+l<5?!p_#$V%QS$R=o{-;2uQnM3$RnJC2xS9u6`2ycTK*gljVd$K`~< zu)tof8NpWMLCL8i#2Ug12vlzJW*!17@<}vG>#!S+Ta68VMqT+oOg10CZs{686Nc~! zQj!szmdOByBC7OuENUudynSqN(=i>W1|=2Vu@a!z%ANikV}p@3n*U<3v2BsTmJ>mB zKMI9J8Nz>Hg+t76IAWPzg4Ci*H%RgpstIJ5PTx2<)VYFfL*67 z0fkwIznyV>2&4^3@yg=={F}%eT%!38z#&LQjm2|c)6|&d=bWbAq3#jS!WogFPS8Oo$Pb9P26xqHB@PV)Rvi|R_SYxYdEQ1)>rPY_@y>$Db#TEFR!)S?!SeO~X3R0Pz_1{^FY%6b zMjPHfR85Lq9-V{6&;n&bI0s+33SYTAVNeZ*saX79z~1aBBemQafr+f0_Yn!ck4N*y z$OfZ8RF25{P1r~j_GYTw5%eI;8ZnMmrc6i|zJQW&*!`0Lp2#)?@6KF(*i=rK0U}6Y z6?@9JtaA_8&EWq~sP3kxfv&5^V5(smDCMU|2%nUvqt`NC(=_@n$}e7j4(= z_``~mi~DY398IiISYFT2f8dBXTd#wGJ<_`y9OvCo=ev}3p)+_vj7t_KLWz& z=^QkvE|3P*3+{|XSB;e-iO2w_MSKpFlOVfzb05~r(554LpA~Iwc@O}eq$GUMdbF-oN_e3^A>Jci^iANQa1sTl&KNDFZ$sm#K=MVEO zVH%!e^65o1ejX`vkcrGa&3~?c;MX!nF=+sq=fenI#ej_i_n{ERli`?eTyAScUeIlK zqI7%SySYb}l-wj8RxR{;^QHv*dauH^)}b~ zFgvy4XDih?{=T1Ro8=$L?C(Ed6J6Q-^j0dir6J;EK6Nc!c#@i|ut(Ic90=d&_QCYo z+)R0ortQ3WJ}u|3bkvOy1?eR!?2ea zaO#j}%U)nTNUDs~vI-T*>)Dc7Y^meQm!Kcw$s8;=2cuXs_+4EOHunZJw9p4?86ruH z(>Yt=ZfDFhSi%01=;7U)8)o?}PrFULEI&zXcYAlyb}Q`o1y82&-HHMMY)@6w8&dnm zeWf8p!RN~$L5MZWJ)`+JuzwgG7c^W@aUsAv?jm8;V1Yn}M4HVP&xbcBYX!elenpAg zXN{J68ZC8SMy=efKMJ)0ySZu&{UD1(>v)0HmOveE`8Id zsYA?cabM{E9hAP4^ZV{w&tjhO6&;fB@$EL5jY&GqxxcEuKThcMvhQTUrkK#)HGp0# z(CCg~4>Rsrv>W9qQUpjVm+Cl54BYE&cRBz&Duf;FY1w}%oGJZ4VRHC3eyN9RfeaeK zVC9jU#EY%dZ>CN$4-xkfzC82o=D22ZU>cC!@upwfJD^|)P_Sh0fgbOX?d)kNkY_JH zUlT8scgA|hO3F&uh#3&wX~Sl*j=_`pzN)T^@~6ye*LAS!AJkdbzu+SUQDIM>yvNZU z)6yw@aYAIF&>U|jtfq7$tmdLg62m>c@2m9L5~gllG-6-=7`sfP*fOK9?vdN5qF>L* z3c*GSU?WdHrMYj>MGws^duKHmAx&F8DDrD3VxV=g9&NS9R+nm|&#cK4j(K&DsrRp@ zZ1O#63U$_?Q_69La{)BYleClOXPxm2LHW5W^>0kejR6!gbq1;|~MA!pN&UG=!v@ z45p>qe%_FLD|L$6JAX0fAFxp)v{4zEz{gBZ=EcM$@8HxqhF?g zcN9C-4U}i~@7c!r1oxrQ^XKFh?7r7<*~}}^c~qpifAoV3B=?5f9tkfk;k~rEbA)`D}G5uEpH?=_LFrA%%I+&KQ%__teu?YnZ~)Z67q5nDdRcbyco z{a`Y81>37JRso9%Wr>8XJpb-M(>zQ@=HBu^Pr>+*a~mV-In4~tiIx`5i6#w;7B+W2 zhxR^MqHxg9WDk3v%fC8HRBxy&g+4OSZ^`0j)tU9 zuJ@ldN|D^urF7;+aD7f`3=DKM^j^^+d1+|1&F3-%N!|k;Tux(3cy7v-j!CDhOigLh z+}=o{*MJ@A_ttw{{`!|Keiy2y1zGfLI4`RHakq2rIk`6skDA^CKEKsmS7A?oHyTMj zP*^m()idO}y_Abx>~aOGt+dMfNi(-fP)@l8R)4>_Kl#UnpGC*M%>g+PIhRuyc?+)Z z+VZJM@0+8g#T*|foH6xg-0Tcio$9er(q%@w{*I-U*|8y_gk?ej3PCg(nP<6o_De=; zYjOeuJWe#Fero(h%)4Y1pL1tF#f$psh+XaxYKTFhWbU0@iuEarnuyZU-abX{!TLTq zYd9b5#T{1d9!J*h*q_uFVY{w)7b{2w&s}xJoR&VNRcrM?a}d|oocO!s&g22-9kErL z?QSluG?Mxu@!oh6ML6V4`~~WupX%aj+Jk3)MX3&7JU?37(D(6U(R=oJRz0eN9K@#e z#8X;60}URF^9Fw$bu-eimQ6WhwZu&!(s9>Yb@vm?d!9x3GCpZ=s+`!r2Iz%fcEQ`u zaeWee;nIFHvcjI862I?>%RRN*Aek@En%SPTwQIXrqN=ve)+c$(N+U0XTws-yEvuLP z)JqNfvArffAa?3NN$Rrh57)jJKHs^`SEC|*`C6q*g|A}$TUvYu%{aU2d(G0XrHWwo zFRJtzHnOFuQcX*fBSYRR|1^pzG>R>;IWUR%JoI4Q$$GUwHHzoc!&BMnYu}y&F$o;F zK}RP2cB;tU8YP3TEdMwmKGUSIr%4#LH#()SdhJ&bv{sxhpCOP$qOX)v z-sj|XwQS+`2t{hoMWgR3SVoMv(T`{Kt0TM=(~_erOw05Se#-l_&h4uCZx^p7cimls z&NaWhUlcJVQxYV-`vo0gy&FL?yf_2O&}0S-?VfGC&$lrww_9;=Uy}x`(;mdkyg7MQ z+S{xIFT0A1#UD-I=xv%5~Guz}=>`bPZSWtFsP<7WnR@I$;h1-MLajrMK zTy~}#?Ni~SefD7;j(xmj&JwYYA9Cg5eJw$dziQQ5QGQQ^nR(!cLU2>=;n^1s_Bz-9 zC}Cci9j6I`twBG_GVWrObB?$FlzdQvCoD|dQyaC9q>$EWN6=5ChkBMrcW9S{6J>8Y zuGjG$)H3YNV5SLo{1@JQkrVm#P3x0+gyulIbSG*G=W|~64~8VB+Z~t`zt_dGrSMhF z`6-WTWp1ix`_Gn~ zEA^~>Iash3h?1|S+S4OS-(oe{uIYgS+2HllUXibhK;{>w?de)smXk->>1d?m%Pr~O zDIq^gXs2DCYK7CM?)s85M7OfG8f7U`QS-Z^QCAo%F!? z;C6=}njWPsm#mf)m2Z$anDl1LlNVM*=O&ND&`bU+ZB5#~d~Y*Je)`f-=lEH1pSsT* zSXChQ4(R#FxvnZG-KCdiA(9G~5mtB=hHsj0s=vYXQFRrDg{#~Y_gUYyd$#V=1qNA$ zU)}AcKxo*#Q(GVD>$bGdS<5gjZy6(YW^0JR<|LsY?MSck79Vy*T=tV|WXGh#E4n*# z$dUA`gd6qemw;~aOzdo#O`8k)BjeR9BTiEOlCN*xTDmG%9*GNivqipd^+(Uy7ebEo z9PFFZaq744N-^slS5)ol-(_EDUL?J{y|-CEi?qXP^2<}IGIRyO3UpB($?}O}PCMUN z6I;!87sHBmYqn*7B}rZb=a z>hG#;Y+km+*5Kw+hU^uamy-wXWh>rR6ty(b;@-G*(JtLX85ApFQ&H&V_4|~+Hdko1 zn$WD4+{`_sc(diYX-;(c#WUA6u-=Ot;_cXMQp=)Ftx2c>isu_V@KhHo~QQsn+W*fj!*g%#k!pEwc0|t-kYS z%fDv57IP`}c8)@IONuTEOl8l^zUgApVZ=1qQBnWthxircPp8Fw=wNOw^sb|B`Xc`B zE-P49D)3&YdxLhxiU7`)5If9sf}OWDA>#Y*eE=^kh~Mp>@f&5~3868@`m zUSsDxW=Qj`we7d4^p(k(=URx(jDh~#T03haPyi~$^UCrz?;}XIthE+xYStpvH+OX= z{tA5&`eR>Yu?OjQwzlV;y+30p#|IMTpITd`{4$x;z6&wqf0>T@*}8K<=_iRU-F4?{R;k-wYfpOoNmlOL#kema zok6#>Y@Ztrp8CE1;EU-;cK!CXm_&J>J5N%{m!mb6eK7?dC2vdQ+y&{$3LzAly@2FrRd6}!ixdY(F$jd zL&NM-SnG1vMY{9Cw_%uFLpi%=_Qi_FToqR^Jh2zy;yy88x=G*9QZBz3P`s9Lw_-`< z>GH47)FOALE2c7A8tXe$YPGk%HOjkYecdN=|0l(HN#RM?8tuNiODlMaboOkqt^e?t z+jL4}3igxSIq2QLxw8ED8!x-93B5 zZsrZYwl^S%zK5Xue6XI$sfk?W#v-o8S2lfjyLvN-p+n0$=?p`we}VKOi`CaMOlR(}n(cgY(f8vsh?M!aSoMS(dKoJ_E9#e4J`mn=wBC-^8~+vBzoP?} zCd+%qr?i$71ef;)MV#NIvaV`9i?u0nz#}PCQ;|rqCFVU8TEX@KPepe|9hE;;yheMr zuAA+=cB5zeo#r{Gp4RsDELqptNjXNZ)h~Ye?U69+KWXLo)lBK-*UEamX+)d)_wigc zSh}v>ziSH)7#`!a6tvp4<(!)w>!=v`UwGG-NdE8aFK`$y3-p~zM+~~3;ohr{rt9p0 z9nQAQhi;|>mq9u1I$W?R)qJ3HPFvJ?B6he($rjG@Tz@?40jXWlF)Q~0NrKbHrz@k}Sa$Dj z*`arJ!o_79?w!_g5h4XW>9TX6v;0!u%)H;V%Cm8E7MS~)1`A-uNo`dBhNg{VAJC0h z&aQp^VT>JC{J=Nz3k4$Yp3HB3@a*(uujyj($I|)jKrBCBH($hGOsGC{v%Ha6-uBmu z;VkW-7yVvETLnTBH=DXSrn@fIc*E5KGUcBT-obclYWRTOx^Q*PK87em^W-vtNH9_l zIO}!#%n^HaN6=IT?YYt~-JOXartG0Nmj-$dr zORZ-dyEFKFq+`fdYz6E;!%o+t8$8 z-tekUGhxNNc1*Xd*9kouladIs)1XPl<6b1JTRkwq?8^q0AE@%L58!}b+(b?`CVc3w zr#75z<>XHK>8D^GVWm0#GvEW2%i=!TUhu&M^yZ}br=(*PCfjQZ!=UsV_@UV=3FRs) zo#>H@dFJPl4QF~=1=^Bc{XQ@mo;F-60OV|%w0BHUCSL)qt{EIkuzDs*Q0njAN+sU} zY@XY-QDsI}|Kw|<&n0`OT)4rCa%>F#B03fPwAJ zn$)EQ7D&DfQsp3VeM3{HM^W)*bX1`W7$>VDUD3j{B^tAFI={}DjoCe{#{_i=SjCyftC&^_JKvcZ_~@77U;j&2R8YQ z)PVE2Y%sK%oFP9I$g+w@7P%GdE})xz@cjUGdYxvl93oi<{#nM`uInEPzt_aqC5V-#dmw=)2?NM|GYd~i87m|n$?g_OWi74-UOAeT z<{otuy>?mWT_84gX9+w8T)6fCJZ6a=Tg-cGqr{WgA3!GIG)MTv4t-+MtAs8?UC0@u zEVriZAg)@0@S3oY*^pC&QTI4o-TCyk`&8Qlg@v0$55A}!5QLLUS&j&C!6yKo#=dOL z#DCaU8;&0?t0DeuApY!=oJl<@K*5h6`(cZzo=n{k2%%txHSZ9K%z-Z_0Y~Z>SUyV* zgAZ*W0D)bGPz-!nZ@b8Q9<@MV3G6@L9xXS7osZs^yLp=SWnhFEtC7AQ^*9fv zC<&D%#{(m;QiBWj)myMs4{_5ZX>^A&-fzitw1)GLbekh)tqiZ0L^LSEnIUS8M2h3L zl+}X^T^C56OrC$r@k(;3p#3cLjW!=OwCcj(GmLat^*UhRs6upLrPmN6l0RECq+7v0 z_k2uqC!(hy+3qlhAwD@mStXKALfd9v%c&Vc{Y~Hd&EUAZpb9=t|XC+6Nmr z!Y%0BKc2xQLBUOd?%UsV>Id=N{X&enPt6=%0IzFj803zY9#ONjeiZUJ0x$AHFFJ{z zgR!?nk)wg*x&CtS1uf%*1PJI>emw1W3Rd_~bp`utgW~Jf3aU z>eit}UY~y*m{T`tOQu=8H2g+^!Rq-UCg6g;$X6TW_{TNFZ(%E0w?Mod15=zV4*no{ z506A)(UpAmtKi28&5-JVt$s*qmm-@&^d8&oG#v@Kuge5Pb>$FSSuVvd_tOJ>+SpUX z$H#B`Il?#W1y^`rd(pyw&f~3cflvn@E?Nk;=bu7rd$T2zKYu0cJi`>frMkx~f<<+Y zp+hdKCP>1Mly03SMJ__ZWgQ;l#`fmD^}+U{i}Z7fULkR`$FL!J8u zMNNt`cfVHAB+U>jJ6`(W=yl>gkyL3}4=Y%l@>27&zc0^ORb4=a31R4HKex6@5A6Sy zJUE~sTpyHe-ON^#;Z)z#zvXvA;qpl!3g{YwkQYXnseL7Ydonlsj$GK2)hJ$q>7GDt zKmHdrDFUe={7-i#B0BG%0`LmnKcmpK;r$u&!RC2?56M6|-rpWXRo>sqC^&lF-z;=q z_#b)N{|`S~$>qoBE0_KvOg{{}UV2L3{M4ZXrw(h_oIDKwhpkasqq1uCnpH~5d)6pv zD5+|!SuLjo|4<705dUc00}hUdj@Wws?*}-yXG_2X%SYaD>WIT(ms1BEoc?@9O=FER W?qhOY}MbX_iQ8K%>M&w(q3%< literal 0 HcmV?d00001 diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml index 0e283dfdfc93ca..a81cf39ce386ff 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml @@ -13,7 +13,7 @@ bootstrap: mcps_location: "bootstrap_mcps/root-user.yaml" - name: data-platforms - version: v1 + version: v2 blocking: true async: false mcps_location: "bootstrap_mcps/data-platforms.yaml" diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml index 0b3d815c710980..2230d552ed4c0e 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml @@ -119,6 +119,16 @@ displayName: Hive type: FILE_SYSTEM logoUrl: "/assets/platforms/hivelogo.png" +- entityUrn: urn:li:dataPlatform:hudi + entityType: dataPlatform + aspectName: dataPlatformInfo + changeType: UPSERT + aspect: + datasetNameDelimiter: "." + name: hudi + displayName: Hudi + type: FILE_SYSTEM + logoUrl: "/assets/platforms/hudilogo.png" - entityUrn: urn:li:dataPlatform:iceberg entityType: dataPlatform aspectName: dataPlatformInfo From 49b6284ebfa6fae65bf463e0eb3218b9793bb1f2 Mon Sep 17 00:00:00 2001 From: Steffen Grohsschmiedt Date: Wed, 4 Dec 2024 01:16:44 +0100 Subject: [PATCH 124/174] fix(airflow): fix AthenaOperator extraction (#11857) Co-authored-by: Harshal Sheth --- .../airflow-plugin/setup.py | 2 +- .../src/datahub_airflow_plugin/_extractors.py | 24 +- .../tests/integration/dags/athena_operator.py | 43 ++ .../goldens/v2_athena_operator.json | 672 ++++++++++++++++++ .../v2_athena_operator_no_dag_listener.json | 672 ++++++++++++++++++ .../tests/integration/test_plugin.py | 29 + 6 files changed, 1440 insertions(+), 2 deletions(-) create mode 100644 metadata-ingestion-modules/airflow-plugin/tests/integration/dags/athena_operator.py create mode 100644 metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json create mode 100644 metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator_no_dag_listener.json diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py index 0d5ceefd989dca..02a0bbb6022e04 100644 --- a/metadata-ingestion-modules/airflow-plugin/setup.py +++ b/metadata-ingestion-modules/airflow-plugin/setup.py @@ -96,7 +96,7 @@ def get_long_description(): *plugins["datahub-kafka"], f"acryl-datahub[testing-utils]{_self_pin}", # Extra requirements for loading our test dags. - "apache-airflow[snowflake]>=2.0.2", + "apache-airflow[snowflake,amazon]>=2.0.2", # A collection of issues we've encountered: # - Connexion's new version breaks Airflow: # See https://github.com/apache/airflow/issues/35234. diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py index de0d4f8711f531..28d5775f61f542 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_extractors.py @@ -50,7 +50,6 @@ def __init__(self): "BigQueryOperator", "BigQueryExecuteQueryOperator", # Athena also does something similar. - "AthenaOperator", "AWSAthenaOperator", # Additional types that OL doesn't support. This is only necessary because # on older versions of Airflow, these operators don't inherit from SQLExecuteQueryOperator. @@ -59,6 +58,8 @@ def __init__(self): for operator in _sql_operator_overrides: self.task_to_extractor.extractors[operator] = GenericSqlExtractor + self.task_to_extractor.extractors["AthenaOperator"] = AthenaOperatorExtractor + self.task_to_extractor.extractors[ "BigQueryInsertJobOperator" ] = BigQueryInsertJobOperatorExtractor @@ -276,6 +277,27 @@ def extract(self) -> Optional[TaskMetadata]: ) +class AthenaOperatorExtractor(BaseExtractor): + def extract(self) -> Optional[TaskMetadata]: + from airflow.providers.amazon.aws.operators.athena import ( + AthenaOperator, # type: ignore + ) + + operator: "AthenaOperator" = self.operator + sql = operator.query + if not sql: + self.log.warning("No query found in AthenaOperator") + return None + + return _parse_sql_into_task_metadata( + self, + sql, + platform="athena", + default_database=None, + default_schema=self.operator.database, + ) + + def _snowflake_default_schema(self: "SnowflakeExtractor") -> Optional[str]: if hasattr(self.operator, "schema") and self.operator.schema is not None: return self.operator.schema diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/athena_operator.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/athena_operator.py new file mode 100644 index 00000000000000..96cdacbbad37dd --- /dev/null +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/athena_operator.py @@ -0,0 +1,43 @@ +from datetime import datetime + +from airflow import DAG +from airflow.providers.amazon.aws.operators.athena import AthenaOperator + +ATHENA_COST_TABLE = "costs" +ATHENA_PROCESSED_TABLE = "processed_costs" + + +def _fake_athena_execute(*args, **kwargs): + pass + + +with DAG( + "athena_operator", + start_date=datetime(2023, 1, 1), + schedule_interval=None, + catchup=False, +) as dag: + # HACK: We don't want to send real requests to Athena. As a workaround, + # we can simply monkey-patch the operator. + AthenaOperator.execute = _fake_athena_execute # type: ignore + + transform_cost_table = AthenaOperator( + aws_conn_id="my_aws", + task_id="transform_cost_table", + database="athena_db", + query=""" + CREATE OR REPLACE TABLE {{ params.out_table_name }} AS + SELECT + id, + month, + total_cost, + area, + total_cost / area as cost_per_area + FROM {{ params.in_table_name }} + """, + params={ + "in_table_name": ATHENA_COST_TABLE, + "out_table_name": ATHENA_PROCESSED_TABLE, + }, + output_location="s3://athena-results-bucket/", + ) diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json new file mode 100644 index 00000000000000..baa738fef7b5fd --- /dev/null +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator.json @@ -0,0 +1,672 @@ +[ +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": { + "_access_control": "None", + "catchup": "False", + "description": "None", + "doc_md": "None", + "fileloc": "", + "is_paused_upon_creation": "None", + "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))", + "tags": "[]", + "timezone": "Timezone('UTC')" + }, + "externalUrl": "http://airflow.example.com/tree?dag_id=athena_operator", + "name": "athena_operator", + "env": "PROD" + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "athena_operator" + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'transform_cost_table'", + "execution_timeout": "None", + "sla": "None", + "task_id": "'transform_cost_table'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table", + "name": "transform_cost_table", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)" + ], + "inputDatajobs": [], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)" + ], + "confidenceScore": 1.0 + } + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.processed_costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "run_id": "manual_run_test", + "duration": "", + "start_date": "", + "end_date": "", + "execution_date": "2023-09-27 21:34:38+00:00", + "try_number": "0", + "max_tries": "0", + "external_executor_id": "None", + "state": "running", + "operator": "AthenaOperator", + "priority_weight": "1", + "log_url": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs", + "orchestrator": "airflow", + "dag_id": "athena_operator", + "task_id": "transform_cost_table" + }, + "externalUrl": "http://airflow.example.com/dags/athena_operator/grid?dag_run_id=manual_run_test&task_id=transform_cost_table&map_index=-1&tab=logs", + "name": "athena_operator_transform_cost_table_manual_run_test", + "type": "BATCH_AD_HOC", + "created": { + "time": 1732719433576, + "actor": "urn:li:corpuser:datahub" + } + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "upstreamInstances": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)" + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)" + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.processed_costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1732719433576, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "STARTED", + "attempt": 1 + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1732719433736, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1732719433736 + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'transform_cost_table'", + "execution_timeout": "None", + "sla": "None", + "task_id": "'transform_cost_table'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table", + "name": "transform_cost_table", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)" + ], + "inputDatajobs": [], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)" + ], + "confidenceScore": 1.0 + } + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.processed_costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1732719433747, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "airflow" + } + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator_no_dag_listener.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator_no_dag_listener.json new file mode 100644 index 00000000000000..c53825a9979e3d --- /dev/null +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v2_athena_operator_no_dag_listener.json @@ -0,0 +1,672 @@ +[ +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": { + "_access_control": "None", + "catchup": "False", + "description": "None", + "doc_md": "None", + "fileloc": "", + "is_paused_upon_creation": "None", + "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))", + "tags": "[]", + "timezone": "Timezone('UTC')" + }, + "externalUrl": "http://airflow.example.com/tree?dag_id=athena_operator", + "name": "athena_operator", + "env": "PROD" + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,athena_operator,prod)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "athena_operator" + } + ] + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'transform_cost_table'", + "execution_timeout": "None", + "sla": "None", + "task_id": "'transform_cost_table'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table", + "name": "transform_cost_table", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)" + ], + "inputDatajobs": [], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)" + ], + "confidenceScore": 1.0 + } + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.processed_costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "run_id": "manual_run_test", + "duration": "", + "start_date": "", + "end_date": "", + "execution_date": "2023-09-27 21:34:38+00:00", + "try_number": "0", + "max_tries": "0", + "external_executor_id": "None", + "state": "running", + "operator": "AthenaOperator", + "priority_weight": "1", + "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=athena_operator&map_index=-1", + "orchestrator": "airflow", + "dag_id": "athena_operator", + "task_id": "transform_cost_table" + }, + "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=transform_cost_table&dag_id=athena_operator&map_index=-1", + "name": "athena_operator_transform_cost_table_manual_run_test", + "type": "BATCH_AD_HOC", + "created": { + "time": 1733121901482, + "actor": "urn:li:corpuser:datahub" + } + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "upstreamInstances": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)" + ] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)" + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.processed_costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1733121901482, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "STARTED", + "attempt": 1 + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1733121901625, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "actor": "urn:li:corpuser:airflow", + "operationType": "CREATE", + "lastUpdatedTimestamp": 1733121901625 + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'transform_cost_table'", + "execution_timeout": "None", + "sla": "None", + "task_id": "'transform_cost_table'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]", + "openlineage_job_facet_sql": "{\"_producer\": \"https://github.com/OpenLineage/OpenLineage/tree/1.22.0/integration/airflow\", \"_schemaURL\": \"https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/SqlJobFacet\", \"query\": \"\\n CREATE OR REPLACE TABLE processed_costs AS\\n SELECT\\n id,\\n month,\\n total_cost,\\n area,\\n total_cost / area as cost_per_area\\n FROM costs\\n \"}" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=athena_operator&_flt_3_task_id=transform_cost_table", + "name": "transform_cost_table", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "dataJobInputOutput", + "aspect": { + "json": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)" + ], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)" + ], + "inputDatajobs": [], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),month)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),month)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),total_cost)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),area)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),area)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD),total_cost)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD),cost_per_area)" + ], + "confidenceScore": 1.0 + } + ] + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:athena,athena_db.processed_costs,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetKey", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:athena", + "name": "athena_db.processed_costs", + "origin": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:airflow", + "type": "DEVELOPER", + "source": { + "type": "SERVICE" + } + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:airflow" + } + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,athena_operator,prod),transform_cost_table)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [] + } + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:9cd4fbcec3a50a4988ffc5841beaf0ad", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1733121901675, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "airflow" + } + } + } +} +] \ No newline at end of file diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py index 3becf10703df6c..75bb43af1a43dd 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py @@ -111,6 +111,24 @@ def _wait_for_dag_finish( raise NotReadyError(f"DAG has not finished yet: {dag_run['state']}") +@tenacity.retry( + reraise=True, + wait=tenacity.wait_fixed(1), + stop=tenacity.stop_after_delay(90), + retry=tenacity.retry_if_exception_type(NotReadyError), +) +def _wait_for_dag_to_load(airflow_instance: AirflowInstance, dag_id: str) -> None: + print("Checking if DAG was loaded") + res = airflow_instance.session.get( + url=f"{airflow_instance.airflow_url}/api/v1/dags", + timeout=5, + ) + res.raise_for_status() + + if len(list(filter(lambda x: x["dag_id"] == dag_id, res.json()["dags"]))) == 0: + raise NotReadyError("DAG was not loaded yet") + + def _dump_dag_logs(airflow_instance: AirflowInstance, dag_id: str) -> None: # Get the dag run info res = airflow_instance.session.get( @@ -206,6 +224,15 @@ def _run_airflow( "insecure_mode": "true", }, ).get_uri(), + "AIRFLOW_CONN_MY_AWS": Connection( + conn_id="my_aws", + conn_type="aws", + extra={ + "region_name": "us-east-1", + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + }, + ).get_uri(), "AIRFLOW_CONN_MY_SQLITE": Connection( conn_id="my_sqlite", conn_type="sqlite", @@ -327,6 +354,7 @@ class DagTestCase: DagTestCase("sqlite_operator", v2_only=True), DagTestCase("custom_operator_dag", v2_only=True), DagTestCase("datahub_emitter_operator_jinja_template_dag", v2_only=True), + DagTestCase("athena_operator", v2_only=True), ] @@ -398,6 +426,7 @@ def test_airflow_plugin( tmp_path, dags_folder=DAGS_FOLDER, is_v1=is_v1 ) as airflow_instance: print(f"Running DAG {dag_id}...") + _wait_for_dag_to_load(airflow_instance, dag_id) subprocess.check_call( [ "airflow", From df9755c9483d9d46603c82b122bbece71dad89be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Wed, 4 Dec 2024 10:06:25 +0100 Subject: [PATCH 125/174] feat(tableau): review reporting and debug traces (#12015) Co-authored-by: Harshal Sheth --- .../ingestion/source/tableau/tableau.py | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 0eafdb4ad23ba0..f3ad5ea706f7ca 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -289,16 +289,12 @@ def make_tableau_client(self, site: str) -> Server: server.auth.sign_in(authentication) return server except ServerResponseError as e: + message = f"Unable to login (invalid/expired credentials or missing permissions): {str(e)}" if isinstance(authentication, PersonalAccessTokenAuth): # Docs on token expiry in Tableau: # https://help.tableau.com/current/server/en-us/security_personal_access_tokens.htm#token-expiry - logger.info( - "Error authenticating with Tableau. Note that Tableau personal access tokens " - "expire if not used for 15 days or if over 1 year old" - ) - raise ValueError( - f"Unable to login (invalid/expired credentials or missing permissions): {str(e)}" - ) from e + message = f"Error authenticating with Tableau. Note that Tableau personal access tokens expire if not used for 15 days or if over 1 year old: {str(e)}" + raise ValueError(message) from e except Exception as e: raise ValueError( f"Unable to login (check your Tableau connection and credentials): {str(e)}" @@ -722,6 +718,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: title="Failed to Retrieve Tableau Metadata", message="Unable to retrieve metadata from tableau.", context=str(md_exception), + exc=md_exception, ) def close(self) -> None: @@ -826,6 +823,7 @@ def _populate_usage_stat_registry(self) -> None: if not view.id: continue self.tableau_stat_registry[view.id] = UsageStat(view_count=view.total_views) + logger.info(f"Got Tableau stats for {len(self.tableau_stat_registry)} assets") logger.debug("Tableau stats %s", self.tableau_stat_registry) def _populate_database_server_hostname_map(self) -> None: @@ -876,7 +874,7 @@ def form_path(project_id: str) -> List[str]: ancestors = [cur_proj.name] while cur_proj.parent_id is not None: if cur_proj.parent_id not in all_project_map: - self.report.report_warning( + self.report.warning( "project-issue", f"Parent project {cur_proj.parent_id} not found. We need Site Administrator Explorer permissions.", ) @@ -974,8 +972,11 @@ def _init_datasource_registry(self) -> None: self.datasource_project_map[ds.id] = ds.project_id except Exception as e: self.report.get_all_datasources_query_failed = True - logger.info(f"Get all datasources query failed due to error {e}") - logger.debug("Error stack trace", exc_info=True) + self.report.warning( + title="Unexpected Query Error", + message="Get all datasources query failed due to error", + exc=e, + ) def _init_workbook_registry(self) -> None: if self.server is None: @@ -1141,7 +1142,6 @@ def get_connection_object_page( ) if node_limit_errors: - logger.debug(f"Node Limit Error. query_data {query_data}") self.report.warning( title="Tableau Data Exceed Predefined Limit", message="The numbers of record in result set exceeds a predefined limit. Increase the tableau " @@ -1257,9 +1257,10 @@ def emit_workbooks(self) -> Iterable[MetadataWorkUnit]: wrk_id: Optional[str] = workbook.get(c.ID) prj_name: Optional[str] = workbook.get(c.PROJECT_NAME) - logger.debug( - f"Skipping workbook {wrk_name}({wrk_id}) as it is project {prj_name}({project_luid}) not " - f"present in project registry" + self.report.warning( + title="Skipping Missing Workbook", + message="Skipping workbook as its project is not present in project registry", + context=f"workbook={wrk_name}({wrk_id}), project={prj_name}({project_luid})", ) continue @@ -1453,7 +1454,7 @@ def get_upstream_tables( c.COLUMNS_CONNECTION ].get("totalCount") if not is_custom_sql and not num_tbl_cols: - logger.debug( + logger.warning( f"Skipping upstream table with id {table[c.ID]}, no columns: {table}" ) continue @@ -1469,7 +1470,12 @@ def get_upstream_tables( table, default_schema_map=self.config.default_schema_map ) except Exception as e: - logger.info(f"Failed to generate upstream reference for {table}: {e}") + self.report.warning( + title="Potentially Missing Lineage Issue", + message="Failed to generate upstream reference", + exc=e, + context=f"table={table}", + ) continue table_urn = ref.make_dataset_urn( @@ -1917,10 +1923,12 @@ def _query_published_datasource_for_project_luid(self, ds_luid: str) -> None: self.datasource_project_map[ds_result.id] = ds_result.project_id except Exception as e: self.report.num_get_datasource_query_failures += 1 - logger.warning( - f"Failed to get datasource project_luid for {ds_luid} due to error {e}" + self.report.warning( + title="Unexpected Query Error", + message="Failed to get datasource details", + exc=e, + context=f"ds_luid={ds_luid}", ) - logger.debug("Error stack trace", exc_info=True) def _get_workbook_project_luid(self, wb: dict) -> Optional[str]: if wb.get(c.LUID) and self.workbook_project_map.get(wb[c.LUID]): From 2b42b29d2fbfb12cfb68a0578b63993bcd182c07 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 4 Dec 2024 04:07:09 -0500 Subject: [PATCH 126/174] fix(ingest/tableau): make `sites.get_by_id` call optional (#12024) --- .../ingestion/source/tableau/tableau.py | 34 ++++++++++++++----- .../tableau/test_tableau_ingest.py | 2 ++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index f3ad5ea706f7ca..197e73dca7141b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -68,6 +68,7 @@ CapabilityReport, MetadataWorkUnitProcessor, Source, + StructuredLogLevel, TestableSource, TestConnectionReport, ) @@ -696,6 +697,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: config=self.config, ctx=self.ctx, site=site, + site_id=site.id, report=self.report, server=self.server, platform=self.platform, @@ -703,11 +705,19 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: logger.info(f"Ingesting assets of site '{site.content_url}'.") yield from site_source.ingest_tableau_site() else: - site = self.server.sites.get_by_id(self.server.site_id) + site = None + with self.report.report_exc( + title="Unable to fetch site details. Site hierarchy may be incomplete and external urls may be missing.", + message="This usually indicates missing permissions. Ensure that you have all necessary permissions.", + level=StructuredLogLevel.WARN, + ): + site = self.server.sites.get_by_id(self.server.site_id) + site_source = TableauSiteSource( config=self.config, ctx=self.ctx, site=site, + site_id=self.server.site_id, report=self.report, server=self.server, platform=self.platform, @@ -740,7 +750,8 @@ def __init__( self, config: TableauConfig, ctx: PipelineContext, - site: SiteItem, + site: Optional[SiteItem], + site_id: Optional[str], report: TableauSourceReport, server: Server, platform: str, @@ -749,9 +760,16 @@ def __init__( self.report = report self.server: Server = server self.ctx: PipelineContext = ctx - self.site: SiteItem = site self.platform = platform + self.site: Optional[SiteItem] = site + if site_id is not None: + self.site_id: str = site_id + else: + assert self.site is not None, "site or site_id is required" + assert self.site.id is not None, "site_id is required when site is provided" + self.site_id = self.site.id + self.database_tables: Dict[str, DatabaseTable] = {} self.tableau_stat_registry: Dict[str, UsageStat] = {} self.tableau_project_registry: Dict[str, TableauProject] = {} @@ -805,7 +823,7 @@ def dataset_browse_prefix(self) -> str: def _re_authenticate(self): tableau_auth: Union[ TableauAuth, PersonalAccessTokenAuth - ] = self.config.get_tableau_auth(self.site.content_url) + ] = self.config.get_tableau_auth(self.site_id) self.server.auth.sign_in(tableau_auth) @property @@ -3189,10 +3207,10 @@ def emit_project_in_topological_order( else: # This is a root Tableau project since the parent_project_id is None. # For a root project, either the site is the parent, or the platform is the default parent. - if self.config.add_site_container and self.site and self.site.id: + if self.config.add_site_container: # The site containers have already been generated by emit_site_container, so we # don't need to emit them again here. - parent_project_key = self.gen_site_key(self.site.id) + parent_project_key = self.gen_site_key(self.site_id) yield from gen_containers( container_key=project_key, @@ -3209,12 +3227,12 @@ def emit_project_in_topological_order( yield from emit_project_in_topological_order(project) def emit_site_container(self): - if not self.site or not self.site.id: + if not self.site: logger.warning("Can not ingest site container. No site information found.") return yield from gen_containers( - container_key=self.gen_site_key(self.site.id), + container_key=self.gen_site_key(self.site_id), name=self.site.name or "Default", sub_types=[c.SITE], ) diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 6c45b8a47de412..38a53b323876d1 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -1028,6 +1028,7 @@ def check_lineage_metadata( ctx=context, platform="tableau", site=SiteItem(name="Site 1", content_url="site1"), + site_id="site1", report=TableauSourceReport(), server=Server("https://test-tableau-server.com"), ) @@ -1248,6 +1249,7 @@ def test_permission_mode_switched_error(pytestconfig, tmp_path, mock_datahub_gra config=mock.MagicMock(), ctx=mock.MagicMock(), site=mock.MagicMock(), + site_id=None, server=mock_sdk.return_value, report=reporter, ) From fcc8b367c1c71e777710b3298f1d0c0c0a362e1e Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 4 Dec 2024 16:24:05 +0530 Subject: [PATCH 127/174] feat(cli): add platform filter for undo soft delete (#12012) --- .../src/datahub/cli/delete_cli.py | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/delete_cli.py b/metadata-ingestion/src/datahub/cli/delete_cli.py index 8b852513e03c0f..a640f941b75276 100644 --- a/metadata-ingestion/src/datahub/cli/delete_cli.py +++ b/metadata-ingestion/src/datahub/cli/delete_cli.py @@ -214,14 +214,47 @@ def references(urn: str, dry_run: bool, force: bool) -> None: @delete.command() -@click.option("--urn", required=True, type=str, help="the urn of the entity") -def undo_by_filter(urn: str) -> None: +@click.option("--urn", required=False, type=str, help="the urn of the entity") +@click.option( + "-p", + "--platform", + required=False, + type=str, + help="Platform filter (e.g. snowflake)", +) +@click.option( + "-b", + "--batch-size", + required=False, + default=3000, + type=int, + help="Batch size when querying for entities to un-soft delete." + "Maximum 10000. Large batch sizes may cause timeouts.", +) +def undo_by_filter( + urn: Optional[str], platform: Optional[str], batch_size: int +) -> None: """ - Undo a soft deletion of an entity + Undo soft deletion by filters """ graph = get_default_graph() logger.info(f"Using {graph}") - graph.set_soft_delete_status(urn=urn, delete=False) + if urn: + graph.set_soft_delete_status(urn=urn, delete=False) + else: + urns = list( + graph.get_urns_by_filter( + platform=platform, + query="*", + status=RemovedStatusFilter.ONLY_SOFT_DELETED, + batch_size=batch_size, + ) + ) + logger.info(f"Going to un-soft delete {len(urns)} urns") + urns_iter = progressbar.progressbar(urns, redirect_stdout=True) + for urn in urns_iter: + assert urn + graph.set_soft_delete_status(urn=urn, delete=False) @delete.command(no_args_is_help=True) From 02815259e0dd0544a9b060f15a61f3ed05820682 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 4 Dec 2024 09:28:40 -0600 Subject: [PATCH 128/174] feat(mcp): add kafka batch processing mode option (#4449) (#12021) Co-authored-by: RyanHolstien --- .../metadata/client/JavaEntityClient.java | 106 +++++++++------- .../client/SystemJavaEntityClient.java | 5 +- .../metadata/client/JavaEntityClientTest.java | 3 +- .../SampleDataFixtureConfiguration.java | 3 +- .../SearchLineageFixtureConfiguration.java | 3 +- .../MetadataChangeProposalsProcessor.java | 80 ++---------- ...BatchMetadataChangeProposalsProcessor.java | 116 ++++++++++++++++++ ...adataChangeProposalProcessorCondition.java | 5 +- ...adataChangeProposalProcessorCondition.java | 16 +++ .../kafka/util/KafkaListenerUtil.java | 96 +++++++++++++++ .../entityclient/JavaEntityClientFactory.java | 4 +- 11 files changed, 312 insertions(+), 125 deletions(-) create mode 100644 metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/batch/BatchMetadataChangeProposalsProcessor.java create mode 100644 metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/batch/BatchMetadataChangeProposalProcessorCondition.java create mode 100644 metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/util/KafkaListenerUtil.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index fa9109689caad4..29faa3955ea662 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -18,6 +18,7 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.EntityClientConfig; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.EnvelopedAspect; import com.linkedin.metadata.aspect.EnvelopedAspectArray; @@ -97,7 +98,7 @@ public class JavaEntityClient implements EntityClient { private final TimeseriesAspectService timeseriesAspectService; private final RollbackService rollbackService; private final EventProducer eventProducer; - private final int batchGetV2Size; + private final EntityClientConfig entityClientConfig; @Override @Nullable @@ -132,7 +133,7 @@ public Map batchGetV2( Map responseMap = new HashMap<>(); - Iterators.partition(urns.iterator(), Math.max(1, batchGetV2Size)) + Iterators.partition(urns.iterator(), Math.max(1, entityClientConfig.getBatchGetV2Size())) .forEachRemaining( batch -> { try { @@ -159,7 +160,8 @@ public Map batchGetVersionedV2( Map responseMap = new HashMap<>(); - Iterators.partition(versionedUrns.iterator(), Math.max(1, batchGetV2Size)) + Iterators.partition( + versionedUrns.iterator(), Math.max(1, entityClientConfig.getBatchGetV2Size())) .forEachRemaining( batch -> { try { @@ -760,48 +762,62 @@ public List batchIngestProposals( : Constants.UNKNOWN_ACTOR; final AuditStamp auditStamp = AuditStampUtils.createAuditStamp(actorUrnStr); - AspectsBatch batch = - AspectsBatchImpl.builder() - .mcps( - metadataChangeProposals, - auditStamp, - opContext.getRetrieverContext().get(), - opContext.getValidationContext().isAlternateValidation()) - .build(); - - List results = entityService.ingestProposal(opContext, batch, async); - entitySearchService.appendRunId(opContext, results); - - Map, List> resultMap = - results.stream() - .collect( - Collectors.groupingBy( - result -> - Pair.of( - result.getRequest().getUrn(), result.getRequest().getAspectName()))); - - // Preserve ordering - return batch.getItems().stream() - .map( - requestItem -> { - // Urns generated - List urnsForRequest = - resultMap - .getOrDefault( - Pair.of(requestItem.getUrn(), requestItem.getAspectName()), List.of()) - .stream() - .map(IngestResult::getUrn) - .filter(Objects::nonNull) - .distinct() - .collect(Collectors.toList()); - - // Update runIds - urnsForRequest.forEach( - urn -> tryIndexRunId(opContext, urn, requestItem.getSystemMetadata())); - - return urnsForRequest.isEmpty() ? null : urnsForRequest.get(0).toString(); - }) - .collect(Collectors.toList()); + List updatedUrns = new ArrayList<>(); + Iterators.partition( + metadataChangeProposals.iterator(), Math.max(1, entityClientConfig.getBatchGetV2Size())) + .forEachRemaining( + batch -> { + AspectsBatch aspectsBatch = + AspectsBatchImpl.builder() + .mcps( + batch, + auditStamp, + opContext.getRetrieverContext().get(), + opContext.getValidationContext().isAlternateValidation()) + .build(); + + List results = + entityService.ingestProposal(opContext, aspectsBatch, async); + entitySearchService.appendRunId(opContext, results); + + Map, List> resultMap = + results.stream() + .collect( + Collectors.groupingBy( + result -> + Pair.of( + result.getRequest().getUrn(), + result.getRequest().getAspectName()))); + + // Preserve ordering + updatedUrns.addAll( + aspectsBatch.getItems().stream() + .map( + requestItem -> { + // Urns generated + List urnsForRequest = + resultMap + .getOrDefault( + Pair.of(requestItem.getUrn(), requestItem.getAspectName()), + List.of()) + .stream() + .map(IngestResult::getUrn) + .filter(Objects::nonNull) + .distinct() + .collect(Collectors.toList()); + + // Update runIds + urnsForRequest.forEach( + urn -> + tryIndexRunId(opContext, urn, requestItem.getSystemMetadata())); + + return urnsForRequest.isEmpty() + ? null + : urnsForRequest.get(0).toString(); + }) + .collect(Collectors.toList())); + }); + return updatedUrns; } @SneakyThrows diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java index ab68abc69bce7c..eda9b3a880228f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/SystemJavaEntityClient.java @@ -5,6 +5,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClientCache; +import com.linkedin.entity.client.EntityClientConfig; import com.linkedin.entity.client.SystemEntityClient; import com.linkedin.metadata.config.cache.client.EntityClientCacheConfig; import com.linkedin.metadata.entity.DeleteEntityService; @@ -43,7 +44,7 @@ public SystemJavaEntityClient( RollbackService rollbackService, EventProducer eventProducer, EntityClientCacheConfig cacheConfig, - int batchGetV2Size) { + EntityClientConfig entityClientConfig) { super( entityService, deleteEntityService, @@ -54,7 +55,7 @@ public SystemJavaEntityClient( timeseriesAspectService, rollbackService, eventProducer, - batchGetV2Size); + entityClientConfig); this.operationContextMap = CacheBuilder.newBuilder().maximumSize(500).build(); this.entityClientCache = buildEntityClientCache(SystemJavaEntityClient.class, cacheConfig); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java b/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java index 7b1fccafbb9e63..4d977d179f91e4 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/client/JavaEntityClientTest.java @@ -12,6 +12,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.RequiredFieldNotPresentException; import com.linkedin.domain.Domains; +import com.linkedin.entity.client.EntityClientConfig; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; import com.linkedin.metadata.aspect.batch.AspectsBatch; @@ -90,7 +91,7 @@ private JavaEntityClient getJavaEntityClient() { _timeseriesAspectService, rollbackService, _eventProducer, - 1); + EntityClientConfig.builder().batchGetV2Size(1).build()); } @Test diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java index e47cdf80281c9a..d5aa7e9c51983a 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SampleDataFixtureConfiguration.java @@ -7,6 +7,7 @@ import static org.mockito.Mockito.when; import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.EntityClientConfig; import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; @@ -330,6 +331,6 @@ private EntityClient entityClientHelper( null, null, null, - 1); + EntityClientConfig.builder().batchGetV2Size(1).build()); } } diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java index 889473d32d1a35..b7b698c73ddac3 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/SearchLineageFixtureConfiguration.java @@ -4,6 +4,7 @@ import static io.datahubproject.test.search.SearchTestUtils.getGraphQueryConfiguration; import com.linkedin.entity.client.EntityClient; +import com.linkedin.entity.client.EntityClientConfig; import com.linkedin.metadata.client.JavaEntityClient; import com.linkedin.metadata.config.DataHubAppConfiguration; import com.linkedin.metadata.config.MetadataChangeProposalConfig; @@ -276,6 +277,6 @@ protected EntityClient entityClient( null, null, null, - 1); + EntityClientConfig.builder().batchGetV2Size(1).build()); } } diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java index ef87afdef46cb7..4e356f5fb3670a 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeProposalsProcessor.java @@ -15,26 +15,21 @@ import com.linkedin.gms.factory.config.ConfigurationProvider; import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; import com.linkedin.metadata.EventUtils; -import com.linkedin.metadata.dao.throttle.ThrottleControl; import com.linkedin.metadata.dao.throttle.ThrottleSensor; import com.linkedin.metadata.kafka.config.MetadataChangeProposalProcessorCondition; +import com.linkedin.metadata.kafka.util.KafkaListenerUtil; import com.linkedin.metadata.utils.metrics.MetricUtils; -import com.linkedin.mxe.FailedMetadataChangeProposal; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.Topics; import io.datahubproject.metadata.context.OperationContext; -import java.io.IOException; import java.util.Optional; -import javax.annotation.Nonnull; import javax.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; import org.slf4j.MDC; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -43,7 +38,6 @@ import org.springframework.kafka.annotation.EnableKafka; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.config.KafkaListenerEndpointRegistry; -import org.springframework.kafka.listener.MessageListenerContainer; import org.springframework.stereotype.Component; @Slf4j @@ -80,38 +74,7 @@ public class MetadataChangeProposalsProcessor { @PostConstruct public void registerConsumerThrottle() { - if (kafkaThrottle != null - && provider - .getMetadataChangeProposal() - .getThrottle() - .getComponents() - .getMceConsumer() - .isEnabled()) { - log.info("MCE Consumer Throttle Enabled"); - kafkaThrottle.addCallback( - (throttleEvent) -> { - Optional container = - Optional.ofNullable(registry.getListenerContainer(mceConsumerGroupId)); - if (container.isEmpty()) { - log.warn( - "Expected container was missing: {} throttle is not possible.", - mceConsumerGroupId); - } else { - if (throttleEvent.isThrottled()) { - container.ifPresent(MessageListenerContainer::pause); - return ThrottleControl.builder() - // resume consumer after sleep - .callback( - (resumeEvent) -> container.ifPresent(MessageListenerContainer::resume)) - .build(); - } - } - - return ThrottleControl.NONE; - }); - } else { - log.info("MCE Consumer Throttle Disabled"); - } + KafkaListenerUtil.registerThrottle(kafkaThrottle, provider, registry, mceConsumerGroupId); } @KafkaListener( @@ -132,7 +95,9 @@ public void consume(final ConsumerRecord consumerRecord) consumerRecord.serializedValueSize(), consumerRecord.timestamp()); - log.debug("Record {}", record); + if (log.isDebugEnabled()) { + log.debug("Record {}", record); + } MetadataChangeProposal event = new MetadataChangeProposal(); try { @@ -148,45 +113,18 @@ public void consume(final ConsumerRecord consumerRecord) MDC.put( MDC_CHANGE_TYPE, Optional.ofNullable(changeType).map(ChangeType::toString).orElse("")); - log.debug("MetadataChangeProposal {}", event); - // TODO: Get this from the event itself. + if (log.isDebugEnabled()) { + log.debug("MetadataChangeProposal {}", event); + } String urn = entityClient.ingestProposal(systemOperationContext, event, false); log.info("Successfully processed MCP event urn: {}", urn); } catch (Throwable throwable) { log.error("MCP Processor Error", throwable); log.error("Message: {}", record); - sendFailedMCP(event, throwable); + KafkaListenerUtil.sendFailedMCP(event, throwable, fmcpTopicName, kafkaProducer); } } finally { MDC.clear(); } } - - private void sendFailedMCP(@Nonnull MetadataChangeProposal event, @Nonnull Throwable throwable) { - final FailedMetadataChangeProposal failedMetadataChangeProposal = - createFailedMCPEvent(event, throwable); - try { - final GenericRecord genericFailedMCERecord = - EventUtils.pegasusToAvroFailedMCP(failedMetadataChangeProposal); - log.debug("Sending FailedMessages to topic - {}", fmcpTopicName); - log.info( - "Error while processing FMCP: FailedMetadataChangeProposal - {}", - failedMetadataChangeProposal); - kafkaProducer.send(new ProducerRecord<>(fmcpTopicName, genericFailedMCERecord)); - } catch (IOException e) { - log.error( - "Error while sending FailedMetadataChangeProposal: Exception - {}, FailedMetadataChangeProposal - {}", - e.getStackTrace(), - failedMetadataChangeProposal); - } - } - - @Nonnull - private FailedMetadataChangeProposal createFailedMCPEvent( - @Nonnull MetadataChangeProposal event, @Nonnull Throwable throwable) { - final FailedMetadataChangeProposal fmcp = new FailedMetadataChangeProposal(); - fmcp.setError(ExceptionUtils.getStackTrace(throwable)); - fmcp.setMetadataChangeProposal(event); - return fmcp; - } } diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/batch/BatchMetadataChangeProposalsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/batch/BatchMetadataChangeProposalsProcessor.java new file mode 100644 index 00000000000000..fed93628fe4d79 --- /dev/null +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/batch/BatchMetadataChangeProposalsProcessor.java @@ -0,0 +1,116 @@ +package com.linkedin.metadata.kafka.batch; + +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.linkedin.entity.client.SystemEntityClient; +import com.linkedin.gms.factory.config.ConfigurationProvider; +import com.linkedin.gms.factory.entityclient.RestliEntityClientFactory; +import com.linkedin.metadata.EventUtils; +import com.linkedin.metadata.dao.throttle.ThrottleSensor; +import com.linkedin.metadata.kafka.config.batch.BatchMetadataChangeProposalProcessorCondition; +import com.linkedin.metadata.kafka.util.KafkaListenerUtil; +import com.linkedin.metadata.utils.metrics.MetricUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.mxe.Topics; +import io.datahubproject.metadata.context.OperationContext; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.Producer; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Import; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@Import({RestliEntityClientFactory.class}) +@Conditional(BatchMetadataChangeProposalProcessorCondition.class) +@EnableKafka +@RequiredArgsConstructor +public class BatchMetadataChangeProposalsProcessor { + private static final String CONSUMER_GROUP_ID_VALUE = + "${METADATA_CHANGE_PROPOSAL_KAFKA_CONSUMER_GROUP_ID:generic-mce-consumer-job-client}"; + + private final OperationContext systemOperationContext; + private final SystemEntityClient entityClient; + private final Producer kafkaProducer; + + @Qualifier("kafkaThrottle") + private final ThrottleSensor kafkaThrottle; + + private final KafkaListenerEndpointRegistry registry; + private final ConfigurationProvider provider; + + private final Histogram kafkaLagStats = + MetricUtils.get().histogram(MetricRegistry.name(this.getClass(), "kafkaLag")); + + @Value( + "${FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME:" + + Topics.FAILED_METADATA_CHANGE_PROPOSAL + + "}") + private String fmcpTopicName; + + @Value(CONSUMER_GROUP_ID_VALUE) + private String mceConsumerGroupId; + + @PostConstruct + public void registerConsumerThrottle() { + KafkaListenerUtil.registerThrottle(kafkaThrottle, provider, registry, mceConsumerGroupId); + } + + @KafkaListener( + id = CONSUMER_GROUP_ID_VALUE, + topics = "${METADATA_CHANGE_PROPOSAL_TOPIC_NAME:" + Topics.METADATA_CHANGE_PROPOSAL + "}", + containerFactory = "kafkaEventConsumer", + batch = "true") + public void consume(final List> consumerRecords) { + try (Timer.Context ignored = MetricUtils.timer(this.getClass(), "consume").time()) { + List metadataChangeProposals = + new ArrayList<>(consumerRecords.size()); + for (ConsumerRecord consumerRecord : consumerRecords) { + kafkaLagStats.update(System.currentTimeMillis() - consumerRecord.timestamp()); + final GenericRecord record = consumerRecord.value(); + + log.info( + "Got MCP event key: {}, topic: {}, partition: {}, offset: {}, value size: {}, timestamp: {}", + consumerRecord.key(), + consumerRecord.topic(), + consumerRecord.partition(), + consumerRecord.offset(), + consumerRecord.serializedValueSize(), + consumerRecord.timestamp()); + + MetadataChangeProposal event = new MetadataChangeProposal(); + try { + event = EventUtils.avroToPegasusMCP(record); + } catch (Throwable throwable) { + log.error("MCP Processor Error", throwable); + log.error("Message: {}", record); + KafkaListenerUtil.sendFailedMCP(event, throwable, fmcpTopicName, kafkaProducer); + } + metadataChangeProposals.add(event); + } + + try { + List urns = + entityClient.batchIngestProposals( + systemOperationContext, metadataChangeProposals, false); + log.info("Successfully processed MCP event urns: {}", urns); + } catch (Exception e) { + // Java client should never throw this + log.error("Exception in batch ingest", e); + } + } + } +} diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java index 1cdb05b04e0ac9..554684d5e8fe77 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/MetadataChangeProposalProcessorCondition.java @@ -9,7 +9,8 @@ public class MetadataChangeProposalProcessorCondition implements Condition { @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { Environment env = context.getEnvironment(); - return "true".equals(env.getProperty("MCE_CONSUMER_ENABLED")) - || "true".equals(env.getProperty("MCP_CONSUMER_ENABLED")); + return ("true".equals(env.getProperty("MCE_CONSUMER_ENABLED")) + || "true".equals(env.getProperty("MCP_CONSUMER_ENABLED"))) + && !Boolean.parseBoolean(env.getProperty("MCP_CONSUMER_BATCH_ENABLED")); } } diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/batch/BatchMetadataChangeProposalProcessorCondition.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/batch/BatchMetadataChangeProposalProcessorCondition.java new file mode 100644 index 00000000000000..296e37c7a90695 --- /dev/null +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/config/batch/BatchMetadataChangeProposalProcessorCondition.java @@ -0,0 +1,16 @@ +package com.linkedin.metadata.kafka.config.batch; + +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +public class BatchMetadataChangeProposalProcessorCondition implements Condition { + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + Environment env = context.getEnvironment(); + return ("true".equals(env.getProperty("MCE_CONSUMER_ENABLED")) + || "true".equals(env.getProperty("MCP_CONSUMER_ENABLED"))) + && Boolean.parseBoolean(env.getProperty("MCP_CONSUMER_BATCH_ENABLED")); + } +} diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/util/KafkaListenerUtil.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/util/KafkaListenerUtil.java new file mode 100644 index 00000000000000..874a45c995e911 --- /dev/null +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/util/KafkaListenerUtil.java @@ -0,0 +1,96 @@ +package com.linkedin.metadata.kafka.util; + +import com.linkedin.gms.factory.config.ConfigurationProvider; +import com.linkedin.metadata.EventUtils; +import com.linkedin.metadata.dao.throttle.ThrottleControl; +import com.linkedin.metadata.dao.throttle.ThrottleSensor; +import com.linkedin.mxe.FailedMetadataChangeProposal; +import com.linkedin.mxe.MetadataChangeProposal; +import java.io.IOException; +import java.util.Optional; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.generic.IndexedRecord; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.kafka.config.KafkaListenerEndpointRegistry; +import org.springframework.kafka.listener.MessageListenerContainer; + +@Slf4j +public class KafkaListenerUtil { + + private KafkaListenerUtil() {} + + public static void registerThrottle( + ThrottleSensor kafkaThrottle, + ConfigurationProvider provider, + KafkaListenerEndpointRegistry registry, + String mceConsumerGroupId) { + if (kafkaThrottle != null + && provider + .getMetadataChangeProposal() + .getThrottle() + .getComponents() + .getMceConsumer() + .isEnabled()) { + log.info("MCE Consumer Throttle Enabled"); + kafkaThrottle.addCallback( + (throttleEvent) -> { + Optional container = + Optional.ofNullable(registry.getListenerContainer(mceConsumerGroupId)); + if (container.isEmpty()) { + log.warn( + "Expected container was missing: {} throttle is not possible.", + mceConsumerGroupId); + } else { + if (throttleEvent.isThrottled()) { + container.ifPresent(MessageListenerContainer::pause); + return ThrottleControl.builder() + // resume consumer after sleep + .callback( + (resumeEvent) -> container.ifPresent(MessageListenerContainer::resume)) + .build(); + } + } + + return ThrottleControl.NONE; + }); + } else { + log.info("MCE Consumer Throttle Disabled"); + } + } + + public static void sendFailedMCP( + @Nonnull MetadataChangeProposal event, + @Nonnull Throwable throwable, + String fmcpTopicName, + Producer kafkaProducer) { + final FailedMetadataChangeProposal failedMetadataChangeProposal = + createFailedMCPEvent(event, throwable); + try { + final GenericRecord genericFailedMCERecord = + EventUtils.pegasusToAvroFailedMCP(failedMetadataChangeProposal); + log.debug("Sending FailedMessages to topic - {}", fmcpTopicName); + log.info( + "Error while processing FMCP: FailedMetadataChangeProposal - {}", + failedMetadataChangeProposal); + kafkaProducer.send(new ProducerRecord<>(fmcpTopicName, genericFailedMCERecord)); + } catch (IOException e) { + log.error( + "Error while sending FailedMetadataChangeProposal: Exception - {}, FailedMetadataChangeProposal - {}", + e.getStackTrace(), + failedMetadataChangeProposal); + } + } + + @Nonnull + public static FailedMetadataChangeProposal createFailedMCPEvent( + @Nonnull MetadataChangeProposal event, @Nonnull Throwable throwable) { + final FailedMetadataChangeProposal fmcp = new FailedMetadataChangeProposal(); + fmcp.setError(ExceptionUtils.getStackTrace(throwable)); + fmcp.setMetadataChangeProposal(event); + return fmcp; + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java index e99978a26d6cf5..e783b4e1963d0a 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/entityclient/JavaEntityClientFactory.java @@ -50,7 +50,7 @@ public EntityClient entityClient( _timeseriesAspectService, rollbackService, _eventProducer, - entityClientConfig.getBatchGetV2Size()); + entityClientConfig); } @Bean("systemEntityClient") @@ -79,6 +79,6 @@ public SystemEntityClient systemEntityClient( rollbackService, _eventProducer, entityClientCacheConfig, - entityClientConfig.getBatchGetV2Size()); + entityClientConfig); } } From 9334b2e324a26e28096c57ddbdcd744aca0a4988 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 4 Dec 2024 21:47:21 +0530 Subject: [PATCH 129/174] chore: update label for team (#12032) --- .github/workflows/pr-labeler.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 7da20ece44f6d6..de7ad21b3e67bb 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -29,7 +29,6 @@ jobs: "swaroopjagadish", "treff7es", "yoonhyejin", - "eboneil", "gabe-lyons", "hsheth2", "jjoyce0510", @@ -37,16 +36,17 @@ jobs: "pedro93", "RyanHolstien", "sakethvarma397", - "Kunal-kankriya", "purnimagarg1", - "dushayntAW", "sagar-salvi-apptware", "kushagra-apptware", "Salman-Apptware", "mayurinehate", "noggi", "skrydal", - "kevinkarchacryl" + "kevinkarchacryl", + "sgomezvillamor", + "acrylJonny", + "chakru-r" ]'), github.actor ) From cbae728cf97e2227ab8e3c140e2f35098c973363 Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 4 Dec 2024 11:30:53 -0800 Subject: [PATCH 130/174] fix(ui): Adding overflow handling (also goes to oss) (#12022) Co-authored-by: John Joyce --- .../ingest/source/executions/reporting/StructuredReportItem.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx index d15f30bc03211c..1cd4349f37d949 100644 --- a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx +++ b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItem.tsx @@ -16,6 +16,7 @@ const StyledCollapse = styled(Collapse)<{ color: string }>` .ant-collapse-header { display: flex; align-items: center; + overflow: auto; } .ant-collapse-item { From ca46c022d5c05f00ba32f37b20691b1155b3e2c8 Mon Sep 17 00:00:00 2001 From: Alice-sky <1835063592@qq.com> Date: Thu, 5 Dec 2024 04:19:18 +0800 Subject: [PATCH 131/174] fix(ingest/pulsar): handle missing/invalid schema objects (#11945) Co-authored-by: Alice --- .../src/datahub/ingestion/source/pulsar.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py index 790c1f918cdfd2..15ee995b2d5fdc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/pulsar.py +++ b/metadata-ingestion/src/datahub/ingestion/source/pulsar.py @@ -78,7 +78,17 @@ class PulsarSchema: def __init__(self, schema): self.schema_version = schema.get("version") - avro_schema = json.loads(schema.get("data")) + schema_data = schema.get("data") + if not schema_data: + logger.warning("Schema data is empty or None. Using default empty schema.") + schema_data = "{}" + + try: + avro_schema = json.loads(schema_data) + except json.JSONDecodeError as e: + logger.error(f"Invalid JSON schema: {schema_data}. Error: {str(e)}") + avro_schema = {} + self.schema_name = avro_schema.get("namespace") + "." + avro_schema.get("name") self.schema_description = avro_schema.get("doc") self.schema_type = schema.get("type") From 97e328260f02c4e9b24ae29321bd6e99390388b8 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 4 Dec 2024 17:12:46 -0500 Subject: [PATCH 132/174] fix(filters) Fix issues with structured properties filters (#11946) --- .../models/StructuredPropertyUtils.java | 2 +- .../request/AggregationQueryBuilder.java | 21 +++- .../metadata/search/utils/ESUtils.java | 25 ++++- .../request/AggregationQueryBuilderTest.java | 104 ++++++++++++++++++ .../metadata/search/utils/ESUtilsTest.java | 43 ++++++++ 5 files changed, 187 insertions(+), 8 deletions(-) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java index 41ef9c25a0f3eb..e9ee7789550c6c 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java @@ -178,7 +178,7 @@ public static String toElasticsearchFieldName( /** * Return an elasticsearch type from structured property type * - * @param fieldName filter or facet field name + * @param fieldName filter or facet field name - must match actual FQN of structured prop * @param aspectRetriever aspect retriever * @return elasticsearch type */ diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java index 39f69ed1716abd..60ca7649331a00 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AggregationQueryBuilder.java @@ -379,7 +379,7 @@ private void addCriteriaFiltersToAggregationMetadata( } } - private void addCriterionFiltersToAggregationMetadata( + public void addCriterionFiltersToAggregationMetadata( @Nonnull final Criterion criterion, @Nonnull final List aggregationMetadata, @Nullable AspectRetriever aspectRetriever) { @@ -422,6 +422,17 @@ private void addCriterionFiltersToAggregationMetadata( value -> addMissingAggregationValueToAggregationMetadata(value, originalAggMetadata)); } + } else if (aggregationMetadataMap.containsKey(criterion.getField())) { + /* + * If we already have aggregations for the facet field (original field name), simply inject any missing values counts into the set. + * If there are no results for a particular facet value, it will NOT be in the original aggregation set returned by + * Elasticsearch. + */ + AggregationMetadata originalAggMetadata = aggregationMetadataMap.get(criterion.getField()); + criterion + .getValues() + .forEach( + value -> addMissingAggregationValueToAggregationMetadata(value, originalAggMetadata)); } else { /* * If we do not have ANY aggregation for the facet field, then inject a new aggregation metadata object for the @@ -429,10 +440,14 @@ private void addCriterionFiltersToAggregationMetadata( * If there are no results for a particular facet, it will NOT be in the original aggregation set returned by * Elasticsearch. */ + // Simply replace suffix from original field when there are no aggregations for it. Prevents + // bug where ES mappings for field are different from how we map the field back to UI + // (ie. Structured Properties with dots in them) + String facetField = ESUtils.replaceSuffix(criterion.getField()); aggregationMetadata.add( buildAggregationMetadata( - finalFacetField, - getFacetToDisplayNames().getOrDefault(finalFacetField, finalFacetField), + facetField, + getFacetToDisplayNames().getOrDefault(facetField, facetField), new LongMap( criterion.getValues().stream().collect(Collectors.toMap(i -> i, i -> 0L))), new FilterValueArray( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 9698a1c10d8b54..17bbbaf059dec4 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -448,9 +448,20 @@ public static String toParentField( urnDefinition.getFirst(), urnDefinition.getSecond())) .orElse(filterField); + return replaceSuffix(fieldName); + } + + /** + * Strip subfields from filter field + * + * @param fieldName name of the field + * @return normalized field name without subfields + */ + @Nonnull + public static String replaceSuffix(@Nonnull final String fieldName) { for (String subfield : SUBFIELDS) { String SUFFIX = "." + subfield; - if (filterField.endsWith(SUFFIX)) { + if (fieldName.endsWith(SUFFIX)) { return fieldName.replace(SUFFIX, ""); } } @@ -710,7 +721,8 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues( final Map> searchableFieldTypes, @Nonnull AspectRetriever aspectRetriever, boolean enableCaseInsensitiveSearch) { - Set fieldTypes = getFieldTypes(searchableFieldTypes, fieldName, aspectRetriever); + Set fieldTypes = + getFieldTypes(searchableFieldTypes, fieldName, criterion, aspectRetriever); if (fieldTypes.size() > 1) { log.warn( "Multiple field types for field name {}, determining best fit for set: {}", @@ -753,12 +765,16 @@ private static QueryBuilder buildEqualsConditionFromCriterionWithValues( private static Set getFieldTypes( Map> searchableFields, String fieldName, + @Nonnull final Criterion criterion, @Nullable AspectRetriever aspectRetriever) { final Set finalFieldTypes; if (fieldName.startsWith(STRUCTURED_PROPERTY_MAPPING_FIELD_PREFIX)) { + // use criterion field here for structured props since fieldName has dots replaced with + // underscores finalFieldTypes = - StructuredPropertyUtils.toElasticsearchFieldType(fieldName, aspectRetriever); + StructuredPropertyUtils.toElasticsearchFieldType( + replaceSuffix(criterion.getField()), aspectRetriever); } else { Set fieldTypes = searchableFields.getOrDefault(fieldName.split("\\.")[0], Collections.emptySet()); @@ -782,7 +798,8 @@ private static RangeQueryBuilder buildRangeQueryFromCriterion( Condition condition, boolean isTimeseries, AspectRetriever aspectRetriever) { - Set fieldTypes = getFieldTypes(searchableFieldTypes, fieldName, aspectRetriever); + Set fieldTypes = + getFieldTypes(searchableFieldTypes, fieldName, criterion, aspectRetriever); // Determine criterion value, range query only accepts single value so take first value in // values if multiple diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java index cef463802a6b14..3969223981ec3f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java @@ -3,6 +3,7 @@ import static com.linkedin.metadata.Constants.DATA_TYPE_URN_PREFIX; import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static com.linkedin.metadata.utils.SearchUtil.*; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -12,23 +13,36 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.LongMap; import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringArray; import com.linkedin.entity.Aspect; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.search.SearchConfiguration; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.annotation.SearchableAnnotation; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.search.AggregationMetadata; +import com.linkedin.metadata.search.FilterValue; +import com.linkedin.metadata.search.FilterValueArray; import com.linkedin.metadata.search.elasticsearch.query.request.AggregationQueryBuilder; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.structured.StructuredPropertyDefinition; import io.datahubproject.test.metadata.context.TestOperationContexts; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import org.mockito.Mockito; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.testng.Assert; @@ -598,4 +612,94 @@ public void testMissingAggregation() { .equals( MISSING_SPECIAL_TYPE + AGGREGATION_SPECIAL_TYPE_DELIMITER + "test"))); } + + @Test + public void testAddFiltersToMetadataWithStructuredPropsNoResults() { + final Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:test_me.one"); + + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = + new AggregationQueryBuilder( + config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of())); + + Criterion criterion = + new Criterion() + .setField("structuredProperties.test_me.one") + .setValues(new StringArray("test123")) + .setCondition(Condition.EQUAL); + + AspectRetriever mockAspectRetriever = getMockAspectRetriever(propertyUrn); + + final List aggregationMetadataList = new ArrayList<>(); + builder.addCriterionFiltersToAggregationMetadata( + criterion, aggregationMetadataList, mockAspectRetriever); + + // ensure we add the correct structured prop aggregation here + Assert.assertEquals(aggregationMetadataList.size(), 1); + // Assert.assertEquals(aggregationMetadataList.get(0).getEntity(), propertyUrn); + Assert.assertEquals( + aggregationMetadataList.get(0).getName(), "structuredProperties.test_me.one"); + Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().size(), 1); + Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().get("test123"), 0); + } + + @Test + public void testAddFiltersToMetadataWithStructuredPropsWithAggregations() { + final Urn propertyUrn = UrnUtils.getUrn("urn:li:structuredProperty:test_me.one"); + + final AggregationMetadata aggregationMetadata = new AggregationMetadata(); + aggregationMetadata.setName("structuredProperties.test_me.one"); + FilterValue filterValue = + new FilterValue().setValue("test123").setFiltered(false).setFacetCount(1); + aggregationMetadata.setFilterValues(new FilterValueArray(filterValue)); + LongMap aggregations = new LongMap(); + aggregations.put("test123", 1L); + aggregationMetadata.setAggregations(aggregations); + + SearchConfiguration config = new SearchConfiguration(); + config.setMaxTermBucketSize(25); + + AggregationQueryBuilder builder = + new AggregationQueryBuilder( + config, ImmutableMap.of(mock(EntitySpec.class), ImmutableList.of())); + + Criterion criterion = + new Criterion() + .setField("structuredProperties.test_me.one") + .setValues(new StringArray("test123")) + .setCondition(Condition.EQUAL); + + AspectRetriever mockAspectRetriever = getMockAspectRetriever(propertyUrn); + + final List aggregationMetadataList = new ArrayList<>(); + aggregationMetadataList.add(aggregationMetadata); + builder.addCriterionFiltersToAggregationMetadata( + criterion, aggregationMetadataList, mockAspectRetriever); + + Assert.assertEquals(aggregationMetadataList.size(), 1); + Assert.assertEquals( + aggregationMetadataList.get(0).getName(), "structuredProperties.test_me.one"); + Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().size(), 1); + Assert.assertEquals(aggregationMetadataList.get(0).getAggregations().get("test123"), 1); + } + + private AspectRetriever getMockAspectRetriever(Urn propertyUrn) { + AspectRetriever mockAspectRetriever = Mockito.mock(AspectRetriever.class); + Map> mockResult = new HashMap<>(); + Map aspectMap = new HashMap<>(); + DataMap definition = new DataMap(); + definition.put("qualifiedName", "test_me.one"); + definition.put("valueType", "urn:li:dataType:datahub.string"); + Aspect definitionAspect = new Aspect(definition); + aspectMap.put(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, definitionAspect); + mockResult.put(propertyUrn, aspectMap); + Set urns = new HashSet<>(); + urns.add(propertyUrn); + Mockito.when(mockAspectRetriever.getLatestAspectObjects(eq(urns), any())) + .thenReturn(mockResult); + + return mockAspectRetriever; + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index 54a9e7d8b47bda..4f2bda39ad2117 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -45,6 +45,7 @@ public static void setup() throws RemoteInvocationException, URISyntaxException Urn abFghTenUrn = Urn.createFromString("urn:li:structuredProperty:ab.fgh.ten"); Urn underscoresAndDotsUrn = Urn.createFromString("urn:li:structuredProperty:under.scores.and.dots_make_a_mess"); + Urn dateWithDotsUrn = Urn.createFromString("urn:li:structuredProperty:date_here.with_dot"); // legacy aspectRetriever = mock(AspectRetriever.class); @@ -64,6 +65,18 @@ public static void setup() throws RemoteInvocationException, URISyntaxException STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, new Aspect(structPropAbFghTenDefinition.data())))); + StructuredPropertyDefinition dateWithDotsDefinition = new StructuredPropertyDefinition(); + dateWithDotsDefinition.setVersion(null, SetMode.REMOVE_IF_NULL); + dateWithDotsDefinition.setValueType(Urn.createFromString(DATA_TYPE_URN_PREFIX + "date")); + dateWithDotsDefinition.setQualifiedName("date_here.with_dot"); + when(aspectRetriever.getLatestAspectObjects(eq(Set.of(dateWithDotsUrn)), anySet())) + .thenReturn( + Map.of( + dateWithDotsUrn, + Map.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new Aspect(dateWithDotsDefinition.data())))); + StructuredPropertyDefinition structPropUnderscoresAndDotsDefinition = new StructuredPropertyDefinition(); structPropUnderscoresAndDotsDefinition.setVersion(null, SetMode.REMOVE_IF_NULL); @@ -895,6 +908,36 @@ public void testGetQueryBuilderFromNamespacedStructPropEqualsValueV1() { Assert.assertEquals(result.toString(), expected); } + @Test + public void testGetQueryBuilderFromDatesWithDots() { + + final Criterion singleValueCriterion = + buildCriterion( + "structuredProperties.date_here.with_dot", Condition.GREATER_THAN, "1731974400000"); + + OperationContext opContext = mock(OperationContext.class); + when(opContext.getAspectRetriever()).thenReturn(aspectRetriever); + QueryBuilder result = + ESUtils.getQueryBuilderFromCriterion( + singleValueCriterion, false, new HashMap<>(), opContext, QueryFilterRewriteChain.EMPTY); + // structuredProperties.date_here_with_dot should not have .keyword at the end since this field + // type is type long for dates + String expected = + "{\n" + + " \"range\" : {\n" + + " \"structuredProperties.date_here_with_dot\" : {\n" + + " \"from\" : 1731974400000,\n" + + " \"to\" : null,\n" + + " \"include_lower\" : false,\n" + + " \"include_upper\" : true,\n" + + " \"boost\" : 1.0,\n" + + " \"_name\" : \"structuredProperties.date_here.with_dot\"\n" + + " }\n" + + " }\n" + + "}"; + Assert.assertEquals(result.toString(), expected); + } + @Test public void testGetQueryBuilderFromStructPropExists() { final Criterion singleValueCriterion = buildExistsCriterion("structuredProperties.ab.fgh.ten"); From 65f44efbad04417d6cc761bde5f81bbe0cb986db Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 4 Dec 2024 17:50:56 -0500 Subject: [PATCH 133/174] fix(ingest): avoid bad IPython version (#12035) --- metadata-ingestion/setup.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index c6d55fb5bcc56e..5ae5438e212c5b 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -142,6 +142,15 @@ # datahub does not depend on traitlets directly but great expectations does. # https://github.com/ipython/traitlets/issues/741 "traitlets!=5.2.2", + # GE depends on IPython - we have no direct dependency on it. + # IPython 8.22.0 added a dependency on traitlets 5.13.x, but only declared a + # version requirement of traitlets>5. + # See https://github.com/ipython/ipython/issues/14352. + # This issue was fixed by https://github.com/ipython/ipython/pull/14353, + # which first appeared in IPython 8.22.1. + # As such, we just need to avoid that version in order to get the + # dependencies that we need. IPython probably should've yanked 8.22.0. + "IPython!=8.22.0", "greenlet", *cachetools_lib, } From 8d15df0c112b412dd89e4dd474534493749cdcb8 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 5 Dec 2024 12:41:43 +0530 Subject: [PATCH 134/174] feat(ingest/kafka): additional validation for oauth_db signature (#11996) --- .../configuration/kafka_consumer_config.py | 32 ++++++++++++++++++- .../tests/integration/kafka/oauth.py | 16 ++++++++++ .../tests/integration/kafka/test_kafka.py | 31 +++++++++++++++++- 3 files changed, 77 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py index cac6bb4996391f..f08c78cadc0b2b 100644 --- a/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py +++ b/metadata-ingestion/src/datahub/configuration/kafka_consumer_config.py @@ -1,3 +1,4 @@ +import inspect import logging from typing import Any, Dict, Optional @@ -34,5 +35,34 @@ def _resolve_oauth_callback(self) -> None: "oauth_cb must be a string representing python function reference " "in the format :." ) + + call_back_fn = import_path(call_back) + self._validate_call_back_fn_signature(call_back_fn) + # Set the callback - self._config[CallableConsumerConfig.CALLBACK_ATTRIBUTE] = import_path(call_back) + self._config[CallableConsumerConfig.CALLBACK_ATTRIBUTE] = call_back_fn + + def _validate_call_back_fn_signature(self, call_back_fn: Any) -> None: + sig = inspect.signature(call_back_fn) + + num_positional_args = len( + [ + param + for param in sig.parameters.values() + if param.kind + in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ) + and param.default == inspect.Parameter.empty + ] + ) + + has_variadic_args = any( + param.kind == inspect.Parameter.VAR_POSITIONAL + for param in sig.parameters.values() + ) + + assert num_positional_args == 1 or ( + has_variadic_args and num_positional_args <= 1 + ), "oauth_cb function must accept single positional argument." diff --git a/metadata-ingestion/tests/integration/kafka/oauth.py b/metadata-ingestion/tests/integration/kafka/oauth.py index 28cfee521d6c0f..81a91fcd5e4069 100644 --- a/metadata-ingestion/tests/integration/kafka/oauth.py +++ b/metadata-ingestion/tests/integration/kafka/oauth.py @@ -12,3 +12,19 @@ def create_token(*args: Any, **kwargs: Any) -> Tuple[str, int]: "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456", 3600, ) + + +def create_token_no_args() -> Tuple[str, int]: + logger.warning(MESSAGE) + return ( + "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456", + 3600, + ) + + +def create_token_only_kwargs(**kwargs: Any) -> Tuple[str, int]: + logger.warning(MESSAGE) + return ( + "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJjbGllbnRfaWQiOiJrYWZrYV9jbGllbnQiLCJleHAiOjE2OTg3NjYwMDB9.dummy_sig_abcdef123456", + 3600, + ) diff --git a/metadata-ingestion/tests/integration/kafka/test_kafka.py b/metadata-ingestion/tests/integration/kafka/test_kafka.py index 7462f177684b7e..bf0ec1845a66c2 100644 --- a/metadata-ingestion/tests/integration/kafka/test_kafka.py +++ b/metadata-ingestion/tests/integration/kafka/test_kafka.py @@ -5,9 +5,10 @@ import yaml from freezegun import freeze_time +from datahub.configuration.common import ConfigurationError from datahub.ingestion.api.source import SourceCapability from datahub.ingestion.run.pipeline import Pipeline -from datahub.ingestion.source.kafka.kafka import KafkaSource +from datahub.ingestion.source.kafka.kafka import KafkaSource, KafkaSourceConfig from tests.integration.kafka import oauth # type: ignore from tests.test_helpers import mce_helpers, test_connection_helpers from tests.test_helpers.click_helpers import run_datahub_cmd @@ -157,3 +158,31 @@ def test_kafka_oauth_callback( assert checks["consumer_oauth_callback"], "Consumer oauth callback not found" assert checks["admin_polling"], "Admin polling was not initiated" assert checks["admin_oauth_callback"], "Admin oauth callback not found" + + +def test_kafka_source_oauth_cb_signature(): + with pytest.raises( + ConfigurationError, + match=("oauth_cb function must accept single positional argument."), + ): + KafkaSourceConfig.parse_obj( + { + "connection": { + "bootstrap": "foobar:9092", + "consumer_config": {"oauth_cb": "oauth:create_token_no_args"}, + } + } + ) + + with pytest.raises( + ConfigurationError, + match=("oauth_cb function must accept single positional argument."), + ): + KafkaSourceConfig.parse_obj( + { + "connection": { + "bootstrap": "foobar:9092", + "consumer_config": {"oauth_cb": "oauth:create_token_only_kwargs"}, + } + } + ) From 3c388a56a5d320d9d8a2a3aef02e6794285cf85c Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Thu, 5 Dec 2024 09:49:44 +0100 Subject: [PATCH 135/174] fix(ingest/gc): Adding test and more checks to gc source (#12027) --- .../source/gc/dataprocess_cleanup.py | 70 +++++++---- metadata-ingestion/tests/unit/test_gc.py | 109 ++++++++++++++++++ 2 files changed, 156 insertions(+), 23 deletions(-) create mode 100644 metadata-ingestion/tests/unit/test_gc.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 90641b7059ca40..3e51b7da9e8be1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -208,22 +208,28 @@ def fetch_dpis(self, job_urn: str, batch_size: int) -> List[dict]: dpis = [] start = 0 while True: - job_query_result = self.ctx.graph.execute_graphql( - DATA_PROCESS_INSTANCES_QUERY, - {"dataJobUrn": job_urn, "start": start, "count": batch_size}, - ) - job_data = job_query_result.get("dataJob") - if not job_data: - raise ValueError(f"Error getting job {job_urn}") - - runs_data = job_data.get("runs") - if not runs_data: - raise ValueError(f"Error getting runs for {job_urn}") - - runs = runs_data.get("runs") - dpis.extend(runs) - start += batch_size - if len(runs) < batch_size: + try: + job_query_result = self.ctx.graph.execute_graphql( + DATA_PROCESS_INSTANCES_QUERY, + {"dataJobUrn": job_urn, "start": start, "count": batch_size}, + ) + job_data = job_query_result.get("dataJob") + if not job_data: + logger.error(f"Error getting job {job_urn}") + break + + runs_data = job_data.get("runs") + if not runs_data: + logger.error(f"Error getting runs for {job_urn}") + break + + runs = runs_data.get("runs") + dpis.extend(runs) + start += batch_size + if len(runs) < batch_size: + break + except Exception as e: + logger.error(f"Exception while fetching DPIs for job {job_urn}: {e}") break return dpis @@ -243,8 +249,12 @@ def keep_last_n_dpi( futures[future] = dpi for future in as_completed(futures): - deleted_count_last_n += 1 - futures[future]["deleted"] = True + try: + future.result() + deleted_count_last_n += 1 + futures[future]["deleted"] = True + except Exception as e: + logger.error(f"Exception while deleting DPI: {e}") if deleted_count_last_n % self.config.batch_size == 0: logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}") @@ -279,7 +289,7 @@ def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None: dpis = self.fetch_dpis(job.urn, self.config.batch_size) dpis.sort( key=lambda x: x["created"]["time"] - if x["created"] and x["created"]["time"] + if "created" in x and "time" in x["created"] else 0, reverse=True, ) @@ -314,15 +324,23 @@ def remove_old_dpis( if dpi.get("deleted"): continue - if dpi["created"]["time"] < retention_time * 1000: + if ( + "created" not in dpi + or "time" not in dpi["created"] + or dpi["created"]["time"] < retention_time * 1000 + ): future = executor.submit( self.delete_entity, dpi["urn"], "dataprocessInstance" ) futures[future] = dpi for future in as_completed(futures): - deleted_count_retention += 1 - futures[future]["deleted"] = True + try: + future.result() + deleted_count_retention += 1 + futures[future]["deleted"] = True + except Exception as e: + logger.error(f"Exception while deleting DPI: {e}") if deleted_count_retention % self.config.batch_size == 0: logger.info( @@ -378,8 +396,11 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: dataFlows[flow.urn] = flow scroll_id: Optional[str] = None + previous_scroll_id: Optional[str] = None + dataJobs: Dict[str, List[DataJobEntity]] = defaultdict(list) deleted_jobs: int = 0 + while True: result = self.ctx.graph.execute_graphql( DATAJOB_QUERY, @@ -426,9 +447,11 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: else: dataJobs[datajob_entity.flow_urn].append(datajob_entity) - if not scroll_id: + if not scroll_id or previous_scroll_id == scroll_id: break + previous_scroll_id = scroll_id + logger.info(f"Deleted {deleted_jobs} DataJobs") # Delete empty dataflows if needed if self.config.delete_empty_data_flows: @@ -443,4 +466,5 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if deleted_jobs % self.config.batch_size == 0: logger.info(f"Deleted {deleted_data_flows} DataFlows") logger.info(f"Deleted {deleted_data_flows} DataFlows") + return [] diff --git a/metadata-ingestion/tests/unit/test_gc.py b/metadata-ingestion/tests/unit/test_gc.py new file mode 100644 index 00000000000000..5429c85dd608dc --- /dev/null +++ b/metadata-ingestion/tests/unit/test_gc.py @@ -0,0 +1,109 @@ +import unittest +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.source.gc.dataprocess_cleanup import ( + DataJobEntity, + DataProcessCleanup, + DataProcessCleanupConfig, + DataProcessCleanupReport, +) + + +class TestDataProcessCleanup(unittest.TestCase): + def setUp(self): + self.ctx = PipelineContext(run_id="test_run") + self.ctx.graph = MagicMock() + self.config = DataProcessCleanupConfig() + self.report = DataProcessCleanupReport() + self.cleanup = DataProcessCleanup( + self.ctx, self.config, self.report, dry_run=True + ) + + @patch( + "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis" + ) + def test_delete_dpi_from_datajobs(self, mock_fetch_dpis): + job = DataJobEntity( + urn="urn:li:dataJob:1", + flow_urn="urn:li:dataFlow:1", + lastIngested=int(datetime.now(timezone.utc).timestamp()), + jobId="job1", + dataPlatformInstance="urn:li:dataPlatformInstance:1", + total_runs=10, + ) + mock_fetch_dpis.return_value = [ + { + "urn": f"urn:li:dataprocessInstance:{i}", + "created": { + "time": int(datetime.now(timezone.utc).timestamp() + i) * 1000 + }, + } + for i in range(10) + ] + self.cleanup.delete_dpi_from_datajobs(job) + self.assertEqual(5, self.report.num_aspects_removed) + + @patch( + "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis" + ) + def test_delete_dpi_from_datajobs_without_dpis(self, mock_fetch_dpis): + job = DataJobEntity( + urn="urn:li:dataJob:1", + flow_urn="urn:li:dataFlow:1", + lastIngested=int(datetime.now(timezone.utc).timestamp()), + jobId="job1", + dataPlatformInstance="urn:li:dataPlatformInstance:1", + total_runs=10, + ) + mock_fetch_dpis.return_value = [] + self.cleanup.delete_dpi_from_datajobs(job) + self.assertEqual(0, self.report.num_aspects_removed) + + @patch( + "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis" + ) + def test_delete_dpi_from_datajobs_without_dpi_created_time(self, mock_fetch_dpis): + job = DataJobEntity( + urn="urn:li:dataJob:1", + flow_urn="urn:li:dataFlow:1", + lastIngested=int(datetime.now(timezone.utc).timestamp()), + jobId="job1", + dataPlatformInstance="urn:li:dataPlatformInstance:1", + total_runs=10, + ) + mock_fetch_dpis.return_value = [ + {"urn": f"urn:li:dataprocessInstance:{i}"} for i in range(10) + ] + [ + { + "urn": "urn:li:dataprocessInstance:11", + "created": {"time": int(datetime.now(timezone.utc).timestamp() * 1000)}, + } + ] + self.cleanup.delete_dpi_from_datajobs(job) + self.assertEqual(10, self.report.num_aspects_removed) + + def test_fetch_dpis(self): + assert self.cleanup.ctx.graph + self.cleanup.ctx.graph = MagicMock() + self.cleanup.ctx.graph.execute_graphql.return_value = { + "dataJob": { + "runs": { + "runs": [ + { + "urn": "urn:li:dataprocessInstance:1", + "created": { + "time": int(datetime.now(timezone.utc).timestamp()) + }, + } + ] + } + } + } + dpis = self.cleanup.fetch_dpis("urn:li:dataJob:1", 10) + self.assertEqual(len(dpis), 1) + + +if __name__ == "__main__": + unittest.main() From 3f3f777c063bb10c3ae8cca1bd13652dd3d23b3d Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 5 Dec 2024 07:44:20 -0600 Subject: [PATCH 136/174] fix(graph-edge): fix graph edge delete exception (#12025) --- .github/workflows/build-and-test.yml | 2 + .../models/extractor/FieldExtractor.java | 29 ++++- .../service/UpdateGraphIndicesService.java | 70 +++++----- .../UpdateGraphIndicesServiceTest.java | 122 ++++++++++++++++++ .../service/UpdateIndicesServiceTest.java | 84 ++++++++++++ 5 files changed, 268 insertions(+), 39 deletions(-) create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 412c962cb6e36f..a5889b2d2f92de 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -75,6 +75,8 @@ jobs: path: | ~/.cache/uv key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }} + - name: Install dependencies + run: ./metadata-ingestion/scripts/install_deps.sh - name: Set up JDK 17 uses: actions/setup-java@v4 with: diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java index bef7782d8f7c9a..f4dc2ec2f0cd56 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/extractor/FieldExtractor.java @@ -14,7 +14,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** Extracts fields from a RecordTemplate based on the appropriate {@link FieldSpec}. */ public class FieldExtractor { @@ -30,15 +30,34 @@ private static long getNumArrayWildcards(PathSpec pathSpec) { // Extract the value of each field in the field specs from the input record public static Map> extractFields( - @Nonnull RecordTemplate record, List fieldSpecs) { - return extractFields(record, fieldSpecs, MAX_VALUE_LENGTH); + @Nullable RecordTemplate record, List fieldSpecs) { + return extractFields(record, fieldSpecs, false); } public static Map> extractFields( - @Nonnull RecordTemplate record, List fieldSpecs, int maxValueLength) { + @Nullable RecordTemplate record, List fieldSpecs, boolean requiredFieldExtract) { + return extractFields(record, fieldSpecs, MAX_VALUE_LENGTH, requiredFieldExtract); + } + + public static Map> extractFields( + @Nullable RecordTemplate record, List fieldSpecs, int maxValueLength) { + return extractFields(record, fieldSpecs, maxValueLength, false); + } + + public static Map> extractFields( + @Nullable RecordTemplate record, + List fieldSpecs, + int maxValueLength, + boolean requiredFieldExtract) { final Map> extractedFields = new HashMap<>(); for (T fieldSpec : fieldSpecs) { - Optional value = RecordUtils.getFieldValue(record, fieldSpec.getPath()); + if (requiredFieldExtract && record == null) { + throw new IllegalArgumentException( + "Field extraction is required and the RecordTemplate is null"); + } + Optional value = + Optional.ofNullable(record) + .flatMap(maybeRecord -> RecordUtils.getFieldValue(maybeRecord, fieldSpec.getPath())); if (!value.isPresent()) { extractedFields.put(fieldSpec, Collections.emptyList()); } else { diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java index ef7f681a81539d..efe073fc00dfdc 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateGraphIndicesService.java @@ -190,7 +190,10 @@ private void handleDeleteChangeEvent( urn.getEntityType(), event.getAspectName())); } - RecordTemplate aspect = event.getRecordTemplate(); + final RecordTemplate aspect = + event.getPreviousRecordTemplate() != null + ? event.getPreviousRecordTemplate() + : event.getRecordTemplate(); Boolean isDeletingKey = event.getAspectName().equals(entitySpec.getKeyAspectName()); if (!aspectSpec.isTimeseries()) { @@ -280,8 +283,8 @@ private Pair, HashMap>> getEdgesAndRelationshipTypes @Nonnull final RecordTemplate aspect, @Nonnull final MetadataChangeLog event, final boolean isNewAspectVersion) { - final List edgesToAdd = new ArrayList<>(); - final HashMap> urnToRelationshipTypesBeingAdded = new HashMap<>(); + final List edges = new ArrayList<>(); + final HashMap> urnToRelationshipTypes = new HashMap<>(); // we need to manually set schemaField <-> schemaField edges for fineGrainedLineage and // inputFields @@ -289,36 +292,28 @@ private Pair, HashMap>> getEdgesAndRelationshipTypes if (aspectSpec.getName().equals(Constants.UPSTREAM_LINEAGE_ASPECT_NAME)) { UpstreamLineage upstreamLineage = new UpstreamLineage(aspect.data()); updateFineGrainedEdgesAndRelationships( - urn, - upstreamLineage.getFineGrainedLineages(), - edgesToAdd, - urnToRelationshipTypesBeingAdded); + urn, upstreamLineage.getFineGrainedLineages(), edges, urnToRelationshipTypes); } else if (aspectSpec.getName().equals(Constants.INPUT_FIELDS_ASPECT_NAME)) { final InputFields inputFields = new InputFields(aspect.data()); - updateInputFieldEdgesAndRelationships( - urn, inputFields, edgesToAdd, urnToRelationshipTypesBeingAdded); + updateInputFieldEdgesAndRelationships(urn, inputFields, edges, urnToRelationshipTypes); } else if (aspectSpec.getName().equals(Constants.DATA_JOB_INPUT_OUTPUT_ASPECT_NAME)) { DataJobInputOutput dataJobInputOutput = new DataJobInputOutput(aspect.data()); updateFineGrainedEdgesAndRelationships( - urn, - dataJobInputOutput.getFineGrainedLineages(), - edgesToAdd, - urnToRelationshipTypesBeingAdded); + urn, dataJobInputOutput.getFineGrainedLineages(), edges, urnToRelationshipTypes); } Map> extractedFields = - FieldExtractor.extractFields(aspect, aspectSpec.getRelationshipFieldSpecs()); + FieldExtractor.extractFields(aspect, aspectSpec.getRelationshipFieldSpecs(), true); for (Map.Entry> entry : extractedFields.entrySet()) { - Set relationshipTypes = - urnToRelationshipTypesBeingAdded.getOrDefault(urn, new HashSet<>()); + Set relationshipTypes = urnToRelationshipTypes.getOrDefault(urn, new HashSet<>()); relationshipTypes.add(entry.getKey().getRelationshipName()); - urnToRelationshipTypesBeingAdded.put(urn, relationshipTypes); + urnToRelationshipTypes.put(urn, relationshipTypes); final List newEdges = GraphIndexUtils.extractGraphEdges(entry, aspect, urn, event, isNewAspectVersion); - edgesToAdd.addAll(newEdges); + edges.addAll(newEdges); } - return Pair.of(edgesToAdd, urnToRelationshipTypesBeingAdded); + return Pair.of(edges, urnToRelationshipTypes); } /** Process snapshot and update graph index */ @@ -433,7 +428,7 @@ private void deleteGraphData( @Nonnull final OperationContext opContext, @Nonnull final Urn urn, @Nonnull final AspectSpec aspectSpec, - @Nonnull final RecordTemplate aspect, + @Nullable final RecordTemplate aspect, @Nonnull final Boolean isKeyAspect, @Nonnull final MetadataChangeLog event) { if (isKeyAspect) { @@ -441,21 +436,28 @@ private void deleteGraphData( return; } - Pair, HashMap>> edgeAndRelationTypes = - getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true); - - final HashMap> urnToRelationshipTypesBeingAdded = - edgeAndRelationTypes.getSecond(); - if (!urnToRelationshipTypesBeingAdded.isEmpty()) { - for (Map.Entry> entry : urnToRelationshipTypesBeingAdded.entrySet()) { - graphService.removeEdgesFromNode( - opContext, - entry.getKey(), - new ArrayList<>(entry.getValue()), - createRelationshipFilter( - new Filter().setOr(new ConjunctiveCriterionArray()), - RelationshipDirection.OUTGOING)); + if (aspect != null) { + Pair, HashMap>> edgeAndRelationTypes = + getEdgesAndRelationshipTypesFromAspect(urn, aspectSpec, aspect, event, true); + + final HashMap> urnToRelationshipTypesBeingRemoved = + edgeAndRelationTypes.getSecond(); + if (!urnToRelationshipTypesBeingRemoved.isEmpty()) { + for (Map.Entry> entry : urnToRelationshipTypesBeingRemoved.entrySet()) { + graphService.removeEdgesFromNode( + opContext, + entry.getKey(), + new ArrayList<>(entry.getValue()), + createRelationshipFilter( + new Filter().setOr(new ConjunctiveCriterionArray()), + RelationshipDirection.OUTGOING)); + } } + } else { + log.warn( + "Insufficient information to perform graph delete. Missing deleted aspect {} for entity {}", + aspectSpec.getName(), + urn); } } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java index 03e381a9059da6..dd02b1fdc9d78d 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateGraphIndicesServiceTest.java @@ -1,6 +1,11 @@ package com.linkedin.metadata.service; +import static com.linkedin.metadata.Constants.CONTAINER_ENTITY_NAME; +import static com.linkedin.metadata.search.utils.QueryUtils.createRelationshipFilter; +import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; @@ -8,9 +13,11 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.testng.Assert.assertEquals; +import com.google.common.collect.ImmutableList; import com.linkedin.common.Status; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.container.Container; import com.linkedin.dataset.DatasetProperties; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.Constants; @@ -21,6 +28,14 @@ import com.linkedin.metadata.graph.elastic.ElasticSearchGraphService; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.LineageRegistry; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.RelationshipDirection; +import com.linkedin.metadata.query.filter.RelationshipFilter; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder; import com.linkedin.metadata.search.elasticsearch.update.ESBulkProcessor; import com.linkedin.metadata.utils.GenericRecordUtils; @@ -29,6 +44,8 @@ import com.linkedin.mxe.MetadataChangeLog; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; +import java.util.List; +import javax.annotation.Nonnull; import org.mockito.ArgumentCaptor; import org.opensearch.index.query.QueryBuilder; import org.opensearch.script.Script; @@ -180,4 +197,109 @@ public void testStatusNoOpEvent() { verifyNoInteractions(mockWriteDAO); } + + @Test + public void testMissingAspectGraphDelete() { + // Test deleting a null aspect + test.handleChangeEvent( + TEST_OP_CONTEXT, + new MetadataChangeLog() + .setChangeType(ChangeType.DELETE) + .setEntityType(TEST_URN.getEntityType()) + .setEntityUrn(TEST_URN) + .setAspectName(Constants.CONTAINER_ASPECT_NAME)); + + // For missing aspects, verify no writes + verifyNoInteractions(mockWriteDAO); + } + + @Test + public void testNodeGraphDelete() { + Urn containerUrn = UrnUtils.getUrn("urn:li:container:foo"); + + // Test deleting container entity + test.handleChangeEvent( + TEST_OP_CONTEXT, + new MetadataChangeLog() + .setChangeType(ChangeType.DELETE) + .setEntityType(CONTAINER_ENTITY_NAME) + .setEntityUrn(containerUrn) + .setAspectName(Constants.CONTAINER_KEY_ASPECT_NAME)); + + // Delete all outgoing edges of this entity + verify(mockWriteDAO, times(1)) + .deleteByQuery( + eq(TEST_OP_CONTEXT), + nullable(String.class), + eq(createUrnFilter(containerUrn)), + nullable(String.class), + eq(new Filter().setOr(new ConjunctiveCriterionArray())), + eq(List.of()), + eq(new RelationshipFilter().setDirection(RelationshipDirection.OUTGOING))); + + // Delete all incoming edges of this entity + verify(mockWriteDAO, times(1)) + .deleteByQuery( + eq(TEST_OP_CONTEXT), + nullable(String.class), + eq(createUrnFilter(containerUrn)), + nullable(String.class), + eq(new Filter().setOr(new ConjunctiveCriterionArray())), + eq(List.of()), + eq(new RelationshipFilter().setDirection(RelationshipDirection.INCOMING))); + + // Delete all edges where this entity is a lifecycle owner + verify(mockWriteDAO, times(1)) + .deleteByQuery( + eq(TEST_OP_CONTEXT), + nullable(String.class), + eq(new Filter().setOr(new ConjunctiveCriterionArray())), + nullable(String.class), + eq(new Filter().setOr(new ConjunctiveCriterionArray())), + eq(List.of()), + eq(new RelationshipFilter().setDirection(RelationshipDirection.INCOMING)), + eq(containerUrn.toString())); + } + + @Test + public void testContainerDelete() { + Urn containerUrn = UrnUtils.getUrn("urn:li:container:foo"); + + // Test deleting a container aspect + test.handleChangeEvent( + TEST_OP_CONTEXT, + new MetadataChangeLog() + .setChangeType(ChangeType.DELETE) + .setEntityType(TEST_URN.getEntityType()) + .setEntityUrn(TEST_URN) + .setAspectName(Constants.CONTAINER_ASPECT_NAME) + .setPreviousAspectValue( + GenericRecordUtils.serializeAspect(new Container().setContainer(containerUrn)))); + + // For container aspects, verify that only edges are removed in both cases + verify(mockWriteDAO, times(1)) + .deleteByQuery( + eq(TEST_OP_CONTEXT), + nullable(String.class), + eq(createUrnFilter(TEST_URN)), + nullable(String.class), + eq(new Filter().setOr(new ConjunctiveCriterionArray())), + eq(List.of("IsPartOf")), + eq( + createRelationshipFilter( + new Filter().setOr(new ConjunctiveCriterionArray()), + RelationshipDirection.OUTGOING))); + } + + private static Filter createUrnFilter(@Nonnull final Urn urn) { + Filter filter = new Filter(); + CriterionArray criterionArray = new CriterionArray(); + Criterion criterion = buildCriterion("urn", Condition.EQUAL, urn.toString()); + criterionArray.add(criterion); + filter.setOr( + new ConjunctiveCriterionArray( + ImmutableList.of(new ConjunctiveCriterion().setAnd(criterionArray)))); + + return filter; + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java new file mode 100644 index 00000000000000..43f8cc0ef191d6 --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java @@ -0,0 +1,84 @@ +package com.linkedin.metadata.service; + +import static com.linkedin.metadata.Constants.CONTAINER_ASPECT_NAME; +import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.verify; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.search.EntitySearchService; +import com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders; +import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; +import com.linkedin.metadata.systemmetadata.SystemMetadataService; +import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.utils.SystemMetadataUtils; +import com.linkedin.mxe.MetadataChangeLog; +import io.datahubproject.metadata.context.OperationContext; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class UpdateIndicesServiceTest { + + @Mock private UpdateGraphIndicesService updateGraphIndicesService; + @Mock private EntitySearchService entitySearchService; + @Mock private TimeseriesAspectService timeseriesAspectService; + @Mock private SystemMetadataService systemMetadataService; + @Mock private SearchDocumentTransformer searchDocumentTransformer; + @Mock private EntityIndexBuilders entityIndexBuilders; + + private OperationContext operationContext; + private UpdateIndicesService updateIndicesService; + + @BeforeMethod + public void setup() { + MockitoAnnotations.openMocks(this); + operationContext = TestOperationContexts.systemContextNoSearchAuthorization(); + updateIndicesService = + new UpdateIndicesService( + updateGraphIndicesService, + entitySearchService, + timeseriesAspectService, + systemMetadataService, + searchDocumentTransformer, + entityIndexBuilders, + "MD5"); + } + + @Test + public void testContainerHandleDeleteEvent() throws Exception { + Urn urn = UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)"); + EntitySpec entitySpec = operationContext.getEntityRegistry().getEntitySpec(DATASET_ENTITY_NAME); + AspectSpec aspectSpec = entitySpec.getAspectSpec(CONTAINER_ASPECT_NAME); + + // Create test data + MetadataChangeLog event = new MetadataChangeLog(); + event.setChangeType(ChangeType.DELETE); + event.setEntityUrn(urn); + event.setAspectName(CONTAINER_ASPECT_NAME); + event.setEntityType(urn.getEntityType()); + event.setSystemMetadata(SystemMetadataUtils.createDefaultSystemMetadata()); + + // Execute Delete + updateIndicesService.handleChangeEvent(operationContext, event); + + // Verify + verify(systemMetadataService).deleteAspect(urn.toString(), CONTAINER_ASPECT_NAME); + verify(searchDocumentTransformer) + .transformAspect( + eq(operationContext), + eq(urn), + nullable(RecordTemplate.class), + eq(aspectSpec), + eq(true)); + verify(updateGraphIndicesService).handleChangeEvent(operationContext, event); + } +} From 48b5a6221c8a203268e8905de3238d9d47411f75 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 5 Dec 2024 11:32:31 -0500 Subject: [PATCH 137/174] feat(ingest): add urn validation test files (#12036) --- .../tests/unit/urns/invalid_urns.txt | 40 +++++++++ .../tests/unit/urns/test_corp_group_urn.py | 10 --- .../tests/unit/urns/test_corpuser_urn.py | 10 --- .../tests/unit/urns/test_data_flow_urn.py | 8 -- .../tests/unit/urns/test_data_job_urn.py | 15 ---- .../urns/test_data_process_instance_urn.py | 10 --- .../tests/unit/urns/test_dataset_urn.py | 20 ----- .../tests/unit/urns/test_domain_urn.py | 8 -- .../tests/unit/urns/test_notebook_urn.py | 10 --- .../tests/unit/urns/test_tag_urn.py | 8 -- .../tests/unit/urns/test_urn.py | 88 +++++++++++-------- .../tests/unit/urns/valid_urns.txt | 24 +++++ 12 files changed, 115 insertions(+), 136 deletions(-) create mode 100644 metadata-ingestion/tests/unit/urns/invalid_urns.txt create mode 100644 metadata-ingestion/tests/unit/urns/valid_urns.txt diff --git a/metadata-ingestion/tests/unit/urns/invalid_urns.txt b/metadata-ingestion/tests/unit/urns/invalid_urns.txt new file mode 100644 index 00000000000000..9ce2c99a1a4ee8 --- /dev/null +++ b/metadata-ingestion/tests/unit/urns/invalid_urns.txt @@ -0,0 +1,40 @@ +# Basic URN format tests +urn:li:abc +urn:li:abc: +urn:li:abc:() +urn:li:abc:(abc,) +urn:li:corpuser:abc) + +# Reserved characters +urn:li:corpuser:foo␟bar +urn:li:tag:a,b,c + +# CorpUser URN tests +urn:li:corpuser:(part1,part2) + +# Dataset URN tests +urn:li:dataset:(urn:li:user:abc,dataset,prod) +urn:li:dataset:(urn:li:user:abc,dataset) +urn:li:dataset:(urn:li:user:abc,dataset,invalidEnv) + +# DataFlow URN tests +urn:li:dataFlow:(airflow,flow_id) + +# DataJob URN tests +urn:li:dataJob:(urn:li:user:abc,job_id) +urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod)) + +# Domain URN tests +urn:li:domain:(part1,part2) + +# Tag URN tests +urn:li:tag:(part1,part2) + +# Notebook URN tests +urn:li:notebook:(part1,part2,part3) + +# CorpGroup URN tests +urn:li:corpGroup:(part1,part2) + +# DataProcessInstance URN tests +urn:li:dataProcessInstance:(part1,part2) diff --git a/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py b/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py index 1897a0e8686f09..4e55e78255d1c1 100644 --- a/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_corp_group_urn.py @@ -3,7 +3,6 @@ import pytest from datahub.utilities.urns.corp_group_urn import CorpGroupUrn -from datahub.utilities.urns.error import InvalidUrnError @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -17,12 +16,3 @@ def test_parse_urn(self) -> None: assert str(corp_group_urn) == corp_group_urn_str assert corp_group_urn == CorpGroupUrn(name="abc") assert corp_group_urn == CorpGroupUrn.create_from_id("abc") - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - CorpGroupUrn.create_from_string( - "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)" - ) - - with self.assertRaises(InvalidUrnError): - CorpGroupUrn.create_from_string("urn:li:corpGroup:(part1,part2)") diff --git a/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py b/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py index 7a2a4f4ff4493c..e4a11b4f404c6e 100644 --- a/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_corpuser_urn.py @@ -3,7 +3,6 @@ import pytest from datahub.utilities.urns.corpuser_urn import CorpuserUrn -from datahub.utilities.urns.error import InvalidUrnError @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -17,12 +16,3 @@ def test_parse_urn(self) -> None: assert str(corpuser_urn) == corpuser_urn_str assert corpuser_urn == CorpuserUrn("abc") assert corpuser_urn == CorpuserUrn.create_from_id("abc") - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - CorpuserUrn.create_from_string( - "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)" - ) - - with self.assertRaises(InvalidUrnError): - CorpuserUrn.create_from_string("urn:li:corpuser:(part1,part2)") diff --git a/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py b/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py index 524411121d418b..edb5563c5b22e3 100644 --- a/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_data_flow_urn.py @@ -3,7 +3,6 @@ import pytest from datahub.utilities.urns.data_flow_urn import DataFlowUrn -from datahub.utilities.urns.error import InvalidUrnError @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -16,10 +15,3 @@ def test_parse_urn(self) -> None: assert data_flow_urn.get_env() == "prod" assert data_flow_urn.__str__() == "urn:li:dataFlow:(airflow,def,prod)" assert data_flow_urn == DataFlowUrn("airflow", "def", "prod") - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - DataFlowUrn.create_from_string("urn:li:abc:(airflow,def,prod)") - - with self.assertRaises(InvalidUrnError): - DataFlowUrn.create_from_string("urn:li:dataFlow:(airflow,flow_id)") diff --git a/metadata-ingestion/tests/unit/urns/test_data_job_urn.py b/metadata-ingestion/tests/unit/urns/test_data_job_urn.py index bf039cd2a91f96..484e5a474c0cd2 100644 --- a/metadata-ingestion/tests/unit/urns/test_data_job_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_data_job_urn.py @@ -4,7 +4,6 @@ from datahub.utilities.urns.data_flow_urn import DataFlowUrn from datahub.utilities.urns.data_job_urn import DataJobUrn -from datahub.utilities.urns.error import InvalidUrnError @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -22,17 +21,3 @@ def test_parse_urn(self) -> None: assert data_job_urn == DataJobUrn( "urn:li:dataFlow:(airflow,flow_id,prod)", "job_id" ) - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - DataJobUrn.create_from_string( - "urn:li:abc:(urn:li:dataFlow:(airflow,flow_id,prod),job_id)" - ) - - with self.assertRaises(InvalidUrnError): - DataJobUrn.create_from_string("urn:li:dataJob:(urn:li:user:abc,job_id)") - - with self.assertRaises(InvalidUrnError): - DataJobUrn.create_from_string( - "urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod))" - ) diff --git a/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py b/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py index a86f8dd99416ff..f9087b19b13c32 100644 --- a/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_data_process_instance_urn.py @@ -3,7 +3,6 @@ import pytest from datahub.utilities.urns.data_process_instance_urn import DataProcessInstanceUrn -from datahub.utilities.urns.error import InvalidUrnError @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -20,12 +19,3 @@ def test_parse_urn(self) -> None: assert dataprocessinstance_urn == DataProcessInstanceUrn("abc") assert dataprocessinstance_urn == DataProcessInstanceUrn.create_from_id("abc") assert "abc" == dataprocessinstance_urn.get_dataprocessinstance_id() - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - DataProcessInstanceUrn.create_from_string("urn:li:abc:dataProcessInstance") - - with self.assertRaises(InvalidUrnError): - DataProcessInstanceUrn.create_from_string( - "urn:li:dataProcessInstance:(part1,part2)" - ) diff --git a/metadata-ingestion/tests/unit/urns/test_dataset_urn.py b/metadata-ingestion/tests/unit/urns/test_dataset_urn.py index 53065143a6ae4f..1be5cd59152009 100644 --- a/metadata-ingestion/tests/unit/urns/test_dataset_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_dataset_urn.py @@ -4,7 +4,6 @@ from datahub.utilities.urns.data_platform_urn import DataPlatformUrn from datahub.utilities.urns.dataset_urn import DatasetUrn -from datahub.utilities.urns.error import InvalidUrnError @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -20,22 +19,3 @@ def test_parse_urn(self) -> None: assert dataset_urn.get_env() == "PROD" assert dataset_urn.__str__() == dataset_urn_str assert dataset_urn == DatasetUrn("urn:li:dataPlatform:abc", "def", "prod") - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - DatasetUrn.create_from_string( - "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)" - ) - - with self.assertRaises(InvalidUrnError): - DatasetUrn.create_from_string( - "urn:li:dataset:(urn:li:user:abc,dataset,prod)" - ) - - with self.assertRaises(InvalidUrnError): - DatasetUrn.create_from_string("urn:li:dataset:(urn:li:user:abc,dataset)") - - with self.assertRaises(InvalidUrnError): - DatasetUrn.create_from_string( - "urn:li:dataset:(urn:li:user:abc,dataset,invalidEnv)" - ) diff --git a/metadata-ingestion/tests/unit/urns/test_domain_urn.py b/metadata-ingestion/tests/unit/urns/test_domain_urn.py index 843a5bf40f5c63..aa5050ce1c030e 100644 --- a/metadata-ingestion/tests/unit/urns/test_domain_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_domain_urn.py @@ -3,7 +3,6 @@ import pytest from datahub.utilities.urns.domain_urn import DomainUrn -from datahub.utilities.urns.error import InvalidUrnError @pytest.mark.filterwarnings("ignore::DeprecationWarning") @@ -17,10 +16,3 @@ def test_parse_urn(self) -> None: assert str(domain_urn) == domain_urn_str assert domain_urn == DomainUrn("abc") assert domain_urn == DomainUrn.create_from_id("abc") - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - DomainUrn.create_from_string("urn:li:abc:domain") - - with self.assertRaises(InvalidUrnError): - DomainUrn.create_from_string("urn:li:domain:(part1,part2)") diff --git a/metadata-ingestion/tests/unit/urns/test_notebook_urn.py b/metadata-ingestion/tests/unit/urns/test_notebook_urn.py index 3ec580f02142b7..6d4dd2ee6fa8c0 100644 --- a/metadata-ingestion/tests/unit/urns/test_notebook_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_notebook_urn.py @@ -2,7 +2,6 @@ import pytest -from datahub.utilities.urns.error import InvalidUrnError from datahub.utilities.urns.notebook_urn import NotebookUrn @@ -16,12 +15,3 @@ def test_parse_urn(self) -> None: assert str(notebook_urn) == notebook_urn_str assert notebook_urn == NotebookUrn("querybook", "123") - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - NotebookUrn.create_from_string( - "urn:li:abc:(urn:li:dataPlatform:abc,def,prod)" - ) - - with self.assertRaises(InvalidUrnError): - NotebookUrn.create_from_string("urn:li:notebook:(part1,part2,part3)") diff --git a/metadata-ingestion/tests/unit/urns/test_tag_urn.py b/metadata-ingestion/tests/unit/urns/test_tag_urn.py index fa3664bcc02180..5f4c9077e28294 100644 --- a/metadata-ingestion/tests/unit/urns/test_tag_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_tag_urn.py @@ -2,7 +2,6 @@ import pytest -from datahub.utilities.urns.error import InvalidUrnError from datahub.utilities.urns.tag_urn import TagUrn @@ -17,10 +16,3 @@ def test_parse_urn(self) -> None: assert str(tag_urn) == tag_urn_str assert tag_urn == TagUrn("abc") assert tag_urn == TagUrn.create_from_id("abc") - - def test_invalid_urn(self) -> None: - with self.assertRaises(InvalidUrnError): - TagUrn.create_from_string("urn:li:abc:tag_id") - - with self.assertRaises(InvalidUrnError): - TagUrn.create_from_string("urn:li:tag:(part1,part2)") diff --git a/metadata-ingestion/tests/unit/urns/test_urn.py b/metadata-ingestion/tests/unit/urns/test_urn.py index 73badb3d1b4234..0c362473c0cf18 100644 --- a/metadata-ingestion/tests/unit/urns/test_urn.py +++ b/metadata-ingestion/tests/unit/urns/test_urn.py @@ -1,16 +1,17 @@ +import logging +import pathlib +from typing import List + import pytest -from datahub.metadata.urns import ( - CorpUserUrn, - DashboardUrn, - DataPlatformUrn, - DatasetUrn, - Urn, -) +from datahub.metadata.urns import CorpUserUrn, DatasetUrn, Urn from datahub.utilities.urns.error import InvalidUrnError pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning") +_CURRENT_DIR = pathlib.Path(__file__).parent +logger = logging.getLogger(__name__) + def test_parse_urn() -> None: simple_urn_str = "urn:li:dataPlatform:abc" @@ -40,38 +41,12 @@ def test_url_encode_urn() -> None: ) -def test_invalid_urn() -> None: - with pytest.raises(InvalidUrnError): - Urn.from_string("urn:li:abc") - - with pytest.raises(InvalidUrnError): - Urn.from_string("urn:li:abc:") - - with pytest.raises(InvalidUrnError): - Urn.from_string("urn:li:abc:()") - - with pytest.raises(InvalidUrnError): - Urn.from_string("urn:li:abc:(abc,)") - - with pytest.raises(InvalidUrnError): - Urn.from_string("urn:li:corpuser:abc)") - - def test_urn_colon() -> None: - # Colon characters are valid in urns, and should not mess up parsing. - - urn = Urn.from_string( - "urn:li:dashboard:(looker,dashboards.thelook::customer_lookup)" - ) - assert isinstance(urn, DashboardUrn) - - assert DataPlatformUrn.from_string("urn:li:dataPlatform:abc:def") - assert DatasetUrn.from_string( - "urn:li:dataset:(urn:li:dataPlatform:abc:def,table_name,PROD)" - ) - assert Urn.from_string("urn:li:corpuser:foo:bar@example.com") + # There's a bunch of other, simpler tests for special characters in the valid_urns test. + # This test ensures that the type dispatch and fields work fine here. # I'm not sure why you'd ever want this, but technically it's a valid urn. + urn = Urn.from_string("urn:li:corpuser::") assert isinstance(urn, CorpUserUrn) assert urn.username == ":" @@ -85,9 +60,48 @@ def test_urn_coercion() -> None: assert urn == Urn.from_string(urn.urn()) -def test_urn_type_dispatch() -> None: +def test_urn_type_dispatch_1() -> None: urn = Urn.from_string("urn:li:dataset:(urn:li:dataPlatform:abc,def,PROD)") assert isinstance(urn, DatasetUrn) with pytest.raises(InvalidUrnError, match="Passed an urn of type corpuser"): DatasetUrn.from_string("urn:li:corpuser:foo") + + +def test_urn_type_dispatch_2() -> None: + urn = "urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod),job_id)" + assert Urn.from_string(urn).urn() == urn + + with pytest.raises(InvalidUrnError, match="Passed an urn of type dataJob"): + CorpUserUrn.from_string(urn) + + +def _load_urns(file_name: pathlib.Path) -> List[str]: + urns = [ + line.strip() + for line in file_name.read_text().splitlines() + if line.strip() and not line.startswith("#") + ] + assert len(urns) > 0, f"No urns found in {file_name}" + return urns + + +def test_valid_urns() -> None: + valid_urns_file = _CURRENT_DIR / "valid_urns.txt" + valid_urns = _load_urns(valid_urns_file) + + for valid_urn in valid_urns: + logger.info(f"Testing valid URN: {valid_urn}") + parsed_urn = Urn.from_string(valid_urn) + assert parsed_urn.urn() == valid_urn + + +def test_invalid_urns() -> None: + invalid_urns_file = _CURRENT_DIR / "invalid_urns.txt" + invalid_urns = _load_urns(invalid_urns_file) + + # Test each invalid URN + for invalid_urn in invalid_urns: + with pytest.raises(InvalidUrnError): + logger.info(f"Testing invalid URN: {invalid_urn}") + Urn.from_string(invalid_urn) diff --git a/metadata-ingestion/tests/unit/urns/valid_urns.txt b/metadata-ingestion/tests/unit/urns/valid_urns.txt new file mode 100644 index 00000000000000..23205ec9a7235b --- /dev/null +++ b/metadata-ingestion/tests/unit/urns/valid_urns.txt @@ -0,0 +1,24 @@ +# Unknown entity types become generic urns +urn:li:abc:foo +urn:li:abc:(foo,bar) +urn:li:abc:(urn:li:dataPlatform:abc,def,prod) + +# A bunch of pretty normal urns +urn:li:corpuser:foo +urn:li:corpGroup:bar +urn:li:dataset:(urn:li:dataPlatform:abc,def/ghi,prod) +urn:li:dataFlow:(airflow,def,prod) +urn:li:dataJob:(urn:li:dataFlow:(airflow,flow_id,prod),job_id) +urn:li:tag:abc +urn:li:chart:(looker,chart_name) +urn:li:dashboard:(looker,dashboard_name) +urn:li:dataProcessInstance:abc +urn:li:domain:abc +urn:li:notebook:(querybook,123) + +# Urns with colons and other special characters +urn:li:tag:dbt:bar +urn:li:tag:: +urn:li:dashboard:(looker,dashboards.thelook::customer_lookup) +urn:li:dataPlatform:abc:def +urn:li:corpuser:foo:bar@example.com From 14fe8891a3c653e17fbfe86bd5fbf80df82e8cbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:40:10 -0600 Subject: [PATCH 138/174] chore(deps): bump cross-spawn from 7.0.3 to 7.0.6 in /datahub-web-react (#11978) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- datahub-web-react/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock index 9dc563c958dd19..ddda98d7f83268 100644 --- a/datahub-web-react/yarn.lock +++ b/datahub-web-react/yarn.lock @@ -5043,9 +5043,9 @@ cross-inspect@1.0.0: tslib "^2.4.0" cross-spawn@^7.0.2: - version "7.0.3" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" - integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" From cb7d68779455cad537710e1fd8bed4a03f6b0405 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 5 Dec 2024 11:45:04 -0600 Subject: [PATCH 139/174] fix(datahub-client): prevent unneeded classes in datahub-client jar (#12037) --- entity-registry/build.gradle | 2 +- .../java/datahub-event/build.gradle | 1 + metadata-models/build.gradle | 15 +++++++++------ .../openapi-servlet/models/build.gradle | 1 + 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle index e5baa95967f304..ee5ece4049399e 100644 --- a/entity-registry/build.gradle +++ b/entity-registry/build.gradle @@ -8,7 +8,7 @@ apply from: "../gradle/coverage/java-coverage.gradle" dependencies { implementation spec.product.pegasus.data - implementation spec.product.pegasus.generator + compileOnly spec.product.pegasus.generator api project(path: ':metadata-models') api project(path: ':metadata-models', configuration: "dataTemplate") api externalDependency.classGraph diff --git a/metadata-integration/java/datahub-event/build.gradle b/metadata-integration/java/datahub-event/build.gradle index 24e119c6229369..3dca2eb0a40c9f 100644 --- a/metadata-integration/java/datahub-event/build.gradle +++ b/metadata-integration/java/datahub-event/build.gradle @@ -18,6 +18,7 @@ dependencies { implementation externalDependency.jacksonDataBind runtimeOnly externalDependency.jna + compileOnly externalDependency.swaggerAnnotations compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok // VisibleForTesting diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index e9379163ecaecc..2d0b433d69013e 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -9,12 +9,15 @@ plugins { apply from: '../gradle/coverage/java-coverage.gradle' dependencies { - api spec.product.pegasus.data - constraints { - implementation('org.apache.commons:commons-text:1.10.0') { - because 'Vulnerability Issue' - } + constraints { + implementation('org.apache.commons:commons-text:1.10.0') { + because 'Vulnerability Issue' } + } + + api(spec.product.pegasus.data) { + exclude group: 'javax.servlet', module: 'javax.servlet-api' + } api project(':li-utils') api project(path: ':li-utils', configuration: "dataTemplate") dataModel project(':li-utils') @@ -26,7 +29,7 @@ dependencies { compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok - api externalDependency.swaggerAnnotations + compileOnly externalDependency.swaggerAnnotations compileOnly externalDependency.jacksonCore compileOnly externalDependency.jacksonDataBind diff --git a/metadata-service/openapi-servlet/models/build.gradle b/metadata-service/openapi-servlet/models/build.gradle index e4100b2d094e04..d75e656e5ecd6c 100644 --- a/metadata-service/openapi-servlet/models/build.gradle +++ b/metadata-service/openapi-servlet/models/build.gradle @@ -10,6 +10,7 @@ dependencies { implementation externalDependency.jacksonDataBind implementation externalDependency.httpClient + compileOnly externalDependency.swaggerAnnotations compileOnly externalDependency.lombok annotationProcessor externalDependency.lombok From c4ada540c4b1460377dd47e8dd6234314c0c7d4c Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 5 Dec 2024 16:02:58 -0600 Subject: [PATCH 140/174] fix(entity-service): no-op batches (#12047) --- .../java/com/linkedin/metadata/Constants.java | 1 + .../metadata/entity/EntityServiceImpl.java | 188 ++++++++++-------- .../entity/EbeanEntityServiceTest.java | 99 ++++++++- 3 files changed, 201 insertions(+), 87 deletions(-) diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 077e0e2b666be1..9c608187342e8c 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -51,6 +51,7 @@ public class Constants { // App sources public static final String UI_SOURCE = "ui"; public static final String SYSTEM_UPDATE_SOURCE = "systemUpdate"; + public static final String METADATA_TESTS_SOURCE = "metadataTests"; /** Entities */ public static final String CORP_USER_ENTITY_NAME = "corpuser"; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index a0a55cf505cf35..bf3481205fb5ab 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -855,6 +855,7 @@ private List ingestAspectsToLocalDB( if (inputBatch.containsDuplicateAspects()) { log.warn(String.format("Batch contains duplicates: %s", inputBatch)); + MetricUtils.counter(EntityServiceImpl.class, "batch_with_duplicate").inc(); } return aspectDao @@ -928,6 +929,7 @@ private List ingestAspectsToLocalDB( // No changes, return if (changeMCPs.isEmpty()) { + MetricUtils.counter(EntityServiceImpl.class, "batch_empty").inc(); return Collections.emptyList(); } @@ -935,6 +937,7 @@ private List ingestAspectsToLocalDB( ValidationExceptionCollection exceptions = AspectsBatch.validatePreCommit(changeMCPs, opContext.getRetrieverContext().get()); if (!exceptions.isEmpty()) { + MetricUtils.counter(EntityServiceImpl.class, "batch_validation_exception").inc(); throw new ValidationException(collectMetrics(exceptions).toString()); } @@ -972,10 +975,13 @@ This condition is specifically for an older conditional write ingestAspectIfNotP */ if (overwrite || databaseAspect == null) { result = - ingestAspectToLocalDB(txContext, writeItem, databaseSystemAspect) - .toBuilder() - .request(writeItem) - .build(); + Optional.ofNullable( + ingestAspectToLocalDB( + txContext, writeItem, databaseSystemAspect)) + .map( + optResult -> + optResult.toBuilder().request(writeItem).build()) + .orElse(null); } else { RecordTemplate oldValue = databaseSystemAspect.getRecordTemplate(); @@ -996,49 +1002,56 @@ This condition is specifically for an older conditional write ingestAspectIfNotP return result; }) + .filter(Objects::nonNull) .collect(Collectors.toList()); - // commit upserts prior to retention or kafka send, if supported by impl - if (txContext != null) { - txContext.commitAndContinue(); - } - long took = TimeUnit.NANOSECONDS.toMillis(ingestToLocalDBTimer.stop()); - if (took > DB_TIMER_LOG_THRESHOLD_MS) { - log.info("Ingestion of aspects batch to database took {} ms", took); - } + if (!upsertResults.isEmpty()) { + // commit upserts prior to retention or kafka send, if supported by impl + if (txContext != null) { + txContext.commitAndContinue(); + } + long took = TimeUnit.NANOSECONDS.toMillis(ingestToLocalDBTimer.stop()); + if (took > DB_TIMER_LOG_THRESHOLD_MS) { + log.info("Ingestion of aspects batch to database took {} ms", took); + } - // Retention optimization and tx - if (retentionService != null) { - List retentionBatch = - upsertResults.stream() - // Only consider retention when there was a previous version - .filter( - result -> - batchAspects.containsKey(result.getUrn().toString()) - && batchAspects - .get(result.getUrn().toString()) - .containsKey(result.getRequest().getAspectName())) - .filter( - result -> { - RecordTemplate oldAspect = result.getOldValue(); - RecordTemplate newAspect = result.getNewValue(); - // Apply retention policies if there was an update to existing aspect - // value - return oldAspect != newAspect - && oldAspect != null - && retentionService != null; - }) - .map( - result -> - RetentionService.RetentionContext.builder() - .urn(result.getUrn()) - .aspectName(result.getRequest().getAspectName()) - .maxVersion(Optional.of(result.getMaxVersion())) - .build()) - .collect(Collectors.toList()); - retentionService.applyRetentionWithPolicyDefaults(opContext, retentionBatch); + // Retention optimization and tx + if (retentionService != null) { + List retentionBatch = + upsertResults.stream() + // Only consider retention when there was a previous version + .filter( + result -> + batchAspects.containsKey(result.getUrn().toString()) + && batchAspects + .get(result.getUrn().toString()) + .containsKey(result.getRequest().getAspectName())) + .filter( + result -> { + RecordTemplate oldAspect = result.getOldValue(); + RecordTemplate newAspect = result.getNewValue(); + // Apply retention policies if there was an update to existing + // aspect + // value + return oldAspect != newAspect + && oldAspect != null + && retentionService != null; + }) + .map( + result -> + RetentionService.RetentionContext.builder() + .urn(result.getUrn()) + .aspectName(result.getRequest().getAspectName()) + .maxVersion(Optional.of(result.getMaxVersion())) + .build()) + .collect(Collectors.toList()); + retentionService.applyRetentionWithPolicyDefaults(opContext, retentionBatch); + } else { + log.warn("Retention service is missing!"); + } } else { - log.warn("Retention service is missing!"); + MetricUtils.counter(EntityServiceImpl.class, "batch_empty_transaction").inc(); + log.warn("Empty transaction detected. {}", inputBatch); } return upsertResults; @@ -2506,7 +2519,7 @@ private Map getEnvelopedAspects( * @param databaseAspect The aspect as it exists in the database. * @return result object */ - @Nonnull + @Nullable private UpdateAspectResult ingestAspectToLocalDB( @Nullable TransactionContext txContext, @Nonnull final ChangeMCP writeItem, @@ -2520,6 +2533,9 @@ private UpdateAspectResult ingestAspectToLocalDB( .setLastRunId(writeItem.getSystemMetadata().getRunId(GetMode.NULL), SetMode.IGNORE_NULL); // 2. Compare the latest existing and new. + final RecordTemplate databaseValue = + databaseAspect == null ? null : databaseAspect.getRecordTemplate(); + final EntityAspect.EntitySystemAspect previousBatchAspect = (EntityAspect.EntitySystemAspect) writeItem.getPreviousSystemAspect(); final RecordTemplate previousValue = @@ -2528,7 +2544,7 @@ private UpdateAspectResult ingestAspectToLocalDB( // 3. If there is no difference between existing and new, we just update // the lastObserved in system metadata. RunId should stay as the original runId if (previousValue != null - && DataTemplateUtil.areEqual(previousValue, writeItem.getRecordTemplate())) { + && DataTemplateUtil.areEqual(databaseValue, writeItem.getRecordTemplate())) { SystemMetadata latestSystemMetadata = previousBatchAspect.getSystemMetadata(); latestSystemMetadata.setLastObserved(writeItem.getSystemMetadata().getLastObserved()); @@ -2564,45 +2580,49 @@ private UpdateAspectResult ingestAspectToLocalDB( } // 4. Save the newValue as the latest version - log.debug( - "Ingesting aspect with name {}, urn {}", writeItem.getAspectName(), writeItem.getUrn()); - String newValueStr = EntityApiUtils.toJsonAspect(writeItem.getRecordTemplate()); - long versionOfOld = - aspectDao.saveLatestAspect( - txContext, - writeItem.getUrn().toString(), - writeItem.getAspectName(), - previousBatchAspect == null ? null : EntityApiUtils.toJsonAspect(previousValue), - previousBatchAspect == null ? null : previousBatchAspect.getCreatedBy(), - previousBatchAspect == null - ? null - : previousBatchAspect.getEntityAspect().getCreatedFor(), - previousBatchAspect == null ? null : previousBatchAspect.getCreatedOn(), - previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadataRaw(), - newValueStr, - writeItem.getAuditStamp().getActor().toString(), - writeItem.getAuditStamp().hasImpersonator() - ? writeItem.getAuditStamp().getImpersonator().toString() - : null, - new Timestamp(writeItem.getAuditStamp().getTime()), - EntityApiUtils.toJsonAspect(writeItem.getSystemMetadata()), - writeItem.getNextAspectVersion()); - - // metrics - aspectDao.incrementWriteMetrics( - writeItem.getAspectName(), 1, newValueStr.getBytes(StandardCharsets.UTF_8).length); - - return UpdateAspectResult.builder() - .urn(writeItem.getUrn()) - .oldValue(previousValue) - .newValue(writeItem.getRecordTemplate()) - .oldSystemMetadata( - previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadata()) - .newSystemMetadata(writeItem.getSystemMetadata()) - .operation(MetadataAuditOperation.UPDATE) - .auditStamp(writeItem.getAuditStamp()) - .maxVersion(versionOfOld) - .build(); + if (!DataTemplateUtil.areEqual(databaseValue, writeItem.getRecordTemplate())) { + log.debug( + "Ingesting aspect with name {}, urn {}", writeItem.getAspectName(), writeItem.getUrn()); + String newValueStr = EntityApiUtils.toJsonAspect(writeItem.getRecordTemplate()); + long versionOfOld = + aspectDao.saveLatestAspect( + txContext, + writeItem.getUrn().toString(), + writeItem.getAspectName(), + previousBatchAspect == null ? null : EntityApiUtils.toJsonAspect(previousValue), + previousBatchAspect == null ? null : previousBatchAspect.getCreatedBy(), + previousBatchAspect == null + ? null + : previousBatchAspect.getEntityAspect().getCreatedFor(), + previousBatchAspect == null ? null : previousBatchAspect.getCreatedOn(), + previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadataRaw(), + newValueStr, + writeItem.getAuditStamp().getActor().toString(), + writeItem.getAuditStamp().hasImpersonator() + ? writeItem.getAuditStamp().getImpersonator().toString() + : null, + new Timestamp(writeItem.getAuditStamp().getTime()), + EntityApiUtils.toJsonAspect(writeItem.getSystemMetadata()), + writeItem.getNextAspectVersion()); + + // metrics + aspectDao.incrementWriteMetrics( + writeItem.getAspectName(), 1, newValueStr.getBytes(StandardCharsets.UTF_8).length); + + return UpdateAspectResult.builder() + .urn(writeItem.getUrn()) + .oldValue(previousValue) + .newValue(writeItem.getRecordTemplate()) + .oldSystemMetadata( + previousBatchAspect == null ? null : previousBatchAspect.getSystemMetadata()) + .newSystemMetadata(writeItem.getSystemMetadata()) + .operation(MetadataAuditOperation.UPDATE) + .auditStamp(writeItem.getAuditStamp()) + .maxVersion(versionOfOld) + .build(); + } + + return null; } private static boolean shouldAspectEmitChangeLog(@Nonnull final AspectSpec aspectSpec) { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index f2ed2fddba7654..a1000fd02abfe1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -1,8 +1,10 @@ package com.linkedin.metadata.entity; +import static com.linkedin.metadata.Constants.APP_SOURCE; import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; import static com.linkedin.metadata.Constants.GLOBAL_TAGS_ASPECT_NAME; +import static com.linkedin.metadata.Constants.METADATA_TESTS_SOURCE; import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; @@ -19,6 +21,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.StringMap; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.AspectGenerationUtils; @@ -61,6 +64,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Triple; import org.testng.Assert; import org.testng.annotations.BeforeMethod; @@ -534,8 +538,8 @@ public void testBatchPatchWithTrailingNoOp() throws Exception { opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); assertEquals( envelopedAspect.getSystemMetadata().getVersion(), - "2", - "Expected version 2. 1 - Initial, + 1 batch operation (1 add, 1 remove)"); + "3", + "Expected version 3. 1 - Initial, + 1 add, 1 remove"); assertEquals( new GlobalTags(envelopedAspect.getValue().data()) .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), @@ -649,7 +653,7 @@ public void testBatchPatchAdd() throws Exception { EnvelopedAspect envelopedAspect = _entityServiceImpl.getLatestEnvelopedAspect( opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); - assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "3", "Expected version 3"); + assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "4", "Expected version 4"); assertEquals( new GlobalTags(envelopedAspect.getValue().data()) .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), @@ -657,6 +661,95 @@ public void testBatchPatchAdd() throws Exception { "Expected all tags"); } + @Test + public void testBatchPatchAddDuplicate() throws Exception { + Urn entityUrn = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)"); + List initialTags = + List.of( + TagUrn.createFromString("urn:li:tag:__default_large_table"), + TagUrn.createFromString("urn:li:tag:__default_low_queries"), + TagUrn.createFromString("urn:li:tag:__default_low_changes"), + TagUrn.createFromString("urn:li:tag:!10TB+ tables")) + .stream() + .map(tag -> new TagAssociation().setTag(tag)) + .collect(Collectors.toList()); + TagUrn tag2 = TagUrn.createFromString("urn:li:tag:$ 1TB+"); + + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + + SystemMetadata patchSystemMetadata = new SystemMetadata(); + patchSystemMetadata.setLastObserved(systemMetadata.getLastObserved() + 1); + patchSystemMetadata.setProperties(new StringMap(Map.of(APP_SOURCE, METADATA_TESTS_SOURCE))); + + ChangeItemImpl initialAspectTag1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .recordTemplate(new GlobalTags().setTags(new TagAssociationArray(initialTags))) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + + PatchItemImpl patchAdd2 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) + .build() + .getJsonPatch()) + .systemMetadata(patchSystemMetadata) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + // establish base entity + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(initialAspectTag1)) + .build(), + false, + true); + + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchAdd2, patchAdd2)) // duplicate + .build(), + false, + true); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "3", "Expected version 3"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Stream.concat(initialTags.stream().map(TagAssociation::getTag), Stream.of(tag2)) + .collect(Collectors.toSet()), + "Expected all tags"); + } + @Test public void dataGeneratorThreadingTest() { DataGenerator dataGenerator = new DataGenerator(opContext, _entityServiceImpl); From b755c6856de65884d7982626574ac9752c8e774f Mon Sep 17 00:00:00 2001 From: Maggie Hays Date: Thu, 5 Dec 2024 16:17:57 -0600 Subject: [PATCH 141/174] docs(compliance-forms) update guide for creating form via UI (#11936) Co-authored-by: yoonhyejin <0327jane@gmail.com> --- docs-website/sidebars.js | 20 +- docs/api/tutorials/forms.md | 10 +- .../compliance-forms/complete-a-form.md | 177 +++++++++++++++++ .../compliance-forms/create-a-form.md | 186 ++++++++++++++++++ .../compliance-forms/overview.md | 46 +++++ .../feature-guides/documentation-forms.md | 113 ----------- docs/features/feature-guides/properties.md | 2 +- 7 files changed, 435 insertions(+), 119 deletions(-) create mode 100644 docs/features/feature-guides/compliance-forms/complete-a-form.md create mode 100644 docs/features/feature-guides/compliance-forms/create-a-form.md create mode 100644 docs/features/feature-guides/compliance-forms/overview.md delete mode 100644 docs/features/feature-guides/documentation-forms.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 3a9d6e10ea8d42..6ae50215c8166f 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -149,6 +149,25 @@ module.exports = { type: "doc", id: "docs/glossary/business-glossary", }, + { + label: "Compliance Forms", + type: "category", + collapsed: true, + items: [ + { + type: "doc", + id: "docs/features/feature-guides/compliance-forms/overview", + }, + { + type: "doc", + id: "docs/features/feature-guides/compliance-forms/create-a-form", + }, + { + type: "doc", + id: "docs/features/feature-guides/compliance-forms/complete-a-form", + }, + ], + }, { label: "Data Contract", type: "doc", @@ -164,7 +183,6 @@ module.exports = { type: "doc", id: "docs/features/dataset-usage-and-query-history", }, - "docs/features/feature-guides/documentation-forms", { label: "Domains", type: "doc", diff --git a/docs/api/tutorials/forms.md b/docs/api/tutorials/forms.md index cf51f1579f1c8a..30dd4db7d8f111 100644 --- a/docs/api/tutorials/forms.md +++ b/docs/api/tutorials/forms.md @@ -1,13 +1,15 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Documentation Forms +# Compliance Forms -## Why Would You Use Documentation Forms? +## Why Would You Use Compliance Forms? -Documentation Forms are a way for end-users to fill out all mandatory attributes associated with a data asset. The form will be dynamically generated based on the definitions provided by administrators and stewards and matching rules. +**DataHub Compliance Forms** streamline the process of documenting, annotating, and classifying your most critical Data Assets through a collaborative, crowdsourced approach. -Learn more about forms in the [Documentation Forms Feature Guide](../../../docs/features/feature-guides/documentation-forms.md). +With Compliance Forms, you can execute large-scale compliance initiatives by assigning tasks (e.g., documentation, tagging, or classification requirements) to the appropriate stakeholders — data owners, stewards, and subject matter experts. + +Learn more about forms in the [Compliance Forms Feature Guide](../../../docs/features/feature-guides/compliance-forms/overview.md). ### Goal Of This Guide This guide will show you how to diff --git a/docs/features/feature-guides/compliance-forms/complete-a-form.md b/docs/features/feature-guides/compliance-forms/complete-a-form.md new file mode 100644 index 00000000000000..285c722179e4d7 --- /dev/null +++ b/docs/features/feature-guides/compliance-forms/complete-a-form.md @@ -0,0 +1,177 @@ +--- +title: Complete a Form +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Complete a DataHub Compliance Form + + +Once a Compliance Form has been published (see [Create a Compliance Form](create-a-form.md)), Assignees will receive notifications in their Task Center prompting them to complete the Form for each Asset they are responsible for. + +This guide provides an example of completing a Compliance Form, covering: + +1. Accessing a Form from an Asset Page or the Task Center +2. Completing a Form for a single Asset or multiple Assets (DataHub Cloud only) +3. Understanding different Form Question completion states + +The example uses the **Governance Initiative 2024**, a Verification Form with 3 Required Questions: + +

+ Sample Compliance Form +

+ +## Access a Compliance Form + +Once you have been assigned to complete a Compliance Form, you will see a **Complete Documentation** or **Complete Verification** option on the right-hand side of an Asset Page: + +

+ Open Compliance Form from Asset Page +

+ +**DataHub Cloud** users can find all outstanding Compliance Form requests by navigating to the **Task Center**: + +

+ Open Compliance Form from Task Center +

+ +## Complete a Form for a Single Asset + +When filling out a Compliance Form for a single Asset, you'll see a list of Questions tailored to that Asset, with clear labels showing which ones are required. Here's how it works: + +- **Question Details:** Each Question specifies if it's required or optional. Required Questions must be completed to submit the Form. +- **Pre-Populated Metadata:** If metadata already exists for a Question, it will appear pre-filled. You can confirm the existing value or make updates as needed. +- **Assignee Contributions:** If another Assignee has already provided a response, their name and the time of submission will be displayed. This gives you visibility into previous input, though you can still update the response. + +:::tip +For Verification Forms, after addressing all required Questions, you'll be prompted to provide final sign-off. This ensures all responses are complete and accurate, marking the Form ready for submission. +::: + +Once you complete all required responses, the sidebar will update with the status of the Asset: + +- **Documented**: All required Questions are completed, Verification is not needed +- **Verified**: All required Questions are completed and Verified + +Here's what the **Governance Initiative 2024** Verification Form looks like for `dogs_in_movies` after responding to all Required Questions: + +

+ Asset Ready to Verify +

+ +And here's the `dogs_in_movies` sidebar after Verifying all responses: + +

+ Asset is Verified +

+ +### Navigate to the Next Asset + +To continue working through the Compliance Forms assigned to you, **use the navigation arrows located in the top-right corner**. These arrows will take you to the next Asset that is still pending Form completion or Verification. Only Assets that require action will appear in this flow, allowing you to focus on the remaining tasks without unnecessary steps. + +## Complete a Form Question for Multiple Assets + +When you want to provide the same response for a question to multiple assets, you can apply it in bulk by selecting the **By Question** option in the top-right corner. This allows you to navigate through the Form question-by-question and apply the same response to multiple assets. + +:::note +Completing Form Questions for multiple Assets is only supported for DataHub Cloud. +::: + +### Example: Apply a Response in Bulk + +Let's look at an example. Imagine we are trying to provide the same answer to a Question for all Assets in a Snowflake schema called `DEMO_DB`. Here's how we'd do it: + +1. **Filter Assets**: Filter down to all datasets in the `DEMO_DB` Snowflake schema. +2. **Set a Response**: For the selected Question, provide a response. In this case, we'll set the Deletion Date to be `2024-12-31`. +3. **Apply to All Selected Assets**: Use the bulk application feature to apply this response to all filtered Assets. + +

+ Apply Response to Multiple Assets +

+ +After setting the response, toggle through each Question, providing the necessary responses to combinations of Assets. + +### Verification for Multiple Assets + +For Verification Forms, as you complete Questions, you will see the number of assets eligible for Verification in the top-right corner. This makes it easy to track which Assets have met the requirements. + +

+ Multiple Assets ready to Verify +

+ +When you are ready to bulk Verify Assets, you will be prompted to confirm that all responses are complete and accurate before proceeding. + +

+ Final Bulk Verification +

+ +### Switch Between Completion Modes + +You can easily toggle between the **Complete By Asset** and **Complete By Question** views as needed, ensuring flexibility while completing and verifying the Compliance Forms. + +## Understanding Different Form Question Completion States + +When completing a Compliance Form, you may encounter various types of Questions, each with unique completion states based on existing metadata or prior responses from other Assignees. This section highlights examples of various completion states to help you understand how Questions can be answered, confirmed, or updated when completing a Form. + +**_1. What is the primary use case for this asset?_** + +This required Question is asking the Assignee to provide Documentation on how the Asset should be used. Note that there is no text populated in the description, meaning the Asset does not have any documentation at all. + +

+ Sample Compliance Form +

+ +**_2. When will this asset be deleted?_** + +You may notice that this question has a pre-populated value. When metadata has been populated from a source _outside_ of a Form, users will have the option to update and save the value, or, simply **Confirm** that the value is accurate. + +

+ Sample Compliance Form +

+ +**_3. Who is the Data Steward of this Asset?_** + +Here's an example where a different Form Assignee has already provided an answer through the Compliance Form 3 days ago. All Assignees will still have the option to update the response, but this allows users to see how other Form Assignees have already answered the questions. + +

+ Sample Compliance Form +

+ + +## FAQ and Troubleshooting + +**Why don’t I see any Compliance Forms in the Task Center or on an Asset Page?** + +If you don’t see any Compliance Forms, check with the Form author to ensure your DataHub user account has been assigned to complete a Form for one or more Assets. Forms can be assigned to Asset Owners, specific DataHub Users, or a combination of both. \ No newline at end of file diff --git a/docs/features/feature-guides/compliance-forms/create-a-form.md b/docs/features/feature-guides/compliance-forms/create-a-form.md new file mode 100644 index 00000000000000..e97aaaa581777d --- /dev/null +++ b/docs/features/feature-guides/compliance-forms/create-a-form.md @@ -0,0 +1,186 @@ +--- +title: Create a Form +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Create a DataHub Compliance Form + + +This guide will walk you through creating and assigning Compliance Forms, including: + +1. Creating a new Compliance Form +2. Building **Questions** for the Compliance Form +3. Assigning **Assets** for the Compliance Form +4. Selecting **Assignees** for the Compliance Form +5. Publishing a Compliance Form + +:::note +Managing Compliance Forms via the DataHub UI is only available in DataHub Cloud. If you are using DataHub Core, please refer to the [Compliance Forms API Guide](../../../api/tutorials/forms.md). +::: + +### Prerequisites + +In order to create, edit, or remove Compliance Forms, you must have the **Manage Compliance Forms** Platform privilege. + +### Step 1: Create a new Compliance Form + +From the navigation bar, head to **Govern** > **Compliance Forms**. Click **+ Create** to start building your Form. + +

+ View of all Compliance Forms +

+ +First up, provide the following details: + +1. **Name:** Select a unique and descriptive name for your Compliance Form that clearly communicates its purpose, such as **"PII Certification Q4 2024"**. + + _**Pro Tip:** This name will be displayed to Assignees when they are assigned tasks, so make it clear and detailed to ensure it conveys the intent of the Form effectively._ + +2. **Description:** Craft a concise yet informative description that explains the purpose of the Compliance Form. Include key details such as the importance of the initiative, its objectives, and the expected completion timeline. This helps Assignees understand the context and significance of their role in the process. + + _**Example:** "This Compliance Form is designed to ensure all datasets containing PII are reviewed and verified by Q4 2024. Completing this Form is critical for compliance with organizational and regulatory requirements."_ + +3. **Type:** Specify the collection type for the Form, based on your compliance requirements: + - **Completion:** The Form is considered complete once all required questions are answered for the selected Assets. We recommend this option for basic requirement completion use cases. + + - **Verification:** The Form is considered complete only when all required questions are answered for the selected Assets **and** an Assignee has explicitly "verified" the responses. We recommend this option when final sign-off by Assignees is necessary, ensuring they acknowledge the accuracy and validity of their responses. + +4. Next, click **Add Question** to begin building the requirements for your Form. + +

+ Create a new Compliance Form +

+ +### Step 2: Build Questions for your Form + +Next, define the Questions for your Compliance Forms. These are used to collect required information about selected assets, and must be completed by an Assignee in order for the Form to be considered complete. + +There are 5 different question types to choose from: + +* **Ownership:** Request one or more owners to be assigned to selected assets. Optionally restrict responses to a specific set of valid users, groups, and ownership types. + * _E.g. Who is responsible for ensuring the accuracy of this Dataset?_ +* **Domain:** Assign a Domain to the Asset, with the option to predefine the set of allowed Domains. + * _E.g. Which Domain does this Dashboard belong to? Sales, Marketing, Finance._ +* **Documentation:** Provide Documentation about the Asset and/or Column. + * _E.g. What is the primary use case of this Dataset? What caveats should others be aware of?_ +* **Glossary Terms:** Assign one or more Glossary Term to the Asset and/or Column, with the option to predefine the set of allowed Glossary Terms. + * _E.g. What types of personally identifiable information (PII) are included in this Asset? Email, Address, SSN, etc._ +* **Structured Properties:** Apply custom properties to an Asset and/or Column. + * _E.g. What date will this Dataset be deprecated and deleted?_ + +When creating a Question, use a clear and concise Title that is easy for Assignees to understand. In the Description, include additional context or instructions to guide their responses. Both the Title and Description will be visible to Assignees when completing the Form, so make sure to provide any specific hints or details they may need to answer the Question accurately and confidently. + +

+ Create a new Compliance Form prompt +

+ +### Step 3: Assign Assets to your Compliance Form + +Now that you have defined the Questions you want Assignees to complete, it's now time to assign the in-scope Assets for this exercise. + +In the **Assign Assets** section, you can easily target the specific set of Assets that are relevant for this Form with the following steps: + +1. Add a Condition or Group of Conditions +2. Choose the appropriate filter type, such as: + * Asset Type (Dataset, Chart, etc.) + * Platform (Snowflake, dbt, etc.) + * Domain (Sales, Marketing, Finance, etc.) + * Assigned Owners + * Assigned Glossary Terms +3. Decide between **All**, **Any**, or **None** of the filters should apply +4. Preview the relevant Assets to confirm you have applied the appropriate filters + +For example, you can apply filters to focus on all **Snowflake Datasets** that are also associated with the **Finance Domain**. This allows you to break down your compliance initiatives into manageable chunks, so you don't have to go after your entire data ecosystem in one go. + +

+ Assign assets to a Compliance Form +

+ +### Step 4: Select Assignees to complete your Compliance Form + +With the Questions and assigned Assets defined, the next step is to select the Assignees—the Users and/or Groups responsible for completing the Form. + +In the **Add Recipients** section, decide who is responsible for completing the Form: + +* **Asset Owners:** Any User that is assigned to one of the in-scope Assets will be able to complete the Form. This is useful for larger initiatives when you may not know the full set of Users. +* **Specific Users and/or Groups:** Select a specific set of Users and/or Groups within DataHub. This is useful when Ownership of the Assets may be poorly-defined. + +

+ Assign recipients to a Compliance Form +

+ +### Step 5: Publish your Form + +After defining the Questions, assigning Assets, and selecting the Assignees, your Form is ready to be published. Once published, Assignees will be notified to complete the Form for the Assets they are responsible for. + + +To publish a Form, simply click **Publish**. + +:::caution +Once you have published a Form, you **cannot** change or add Questions. You can, however, change the set of Assets and/or Assignees for the Form. +::: + +Not ready for primetime just yet? No worries! You also have the option to **Save Draft**. + +

+ Publish a Compliance Form +

+ +## FAQ and Troubleshooting + +**Does answering a Compliance Form Question update the selected Asset?** + +Yes! Compliance Forms serve as a powerful tool for gathering and updating key attributes for your mission-critical Data Assets at scale. When a Question is answered, the response directly updates the corresponding attributes of the selected Asset. + +**How does a Compliance Form interact with existing metadata?** + +If an Asset already has existing metadata that is also referenced in a Form Question, Assignees will have the option to confirm the existing value, overwrite the value, or append additional details. + +_You can find more details and examples in the [Complete a Form](complete-a-form.md#understanding-different-form-question-completion-states) guide._ + +**What is the difference between Completion and Verification Forms?** + +Both Form types are a way to configure a set of optional and/or required Questions for DataHub users to complete. When using Verification Forms, users will be presented with a final verification step once all required questions have been completed; you can think of this as a final acknowledgment of the accuracy of information submitted. + +**Can I assign multiple Forms to a single Asset?** + +You sure can! Please keep in mind that an Asset will only be considered Documented or Verified if all required questions are completed on all assigned Forms. + +**How will DataHub Users know that a Compliance Form has been assigned to them?** + +They have to check the Inbox on the navigation bar. There are no off-platform notifications for Compliance Forms at this time. + +**How do I track the progress of Form completion?** + +Great question. We are working on Compliance Forms Analytics that will directly show you the progress of your initiative across the selected Assets. Stay tuned! + +### API Tutorials + +- [API Guides on Documentation Form](../../../api/tutorials/forms.md) + +### Related Features + +- [DataHub Properties](../../feature-guides/properties.md) + +## Next Steps + +Now that you have created a DataHub Compliance Form, you're ready to [Complete a Compliance Form](complete-a-form.md). \ No newline at end of file diff --git a/docs/features/feature-guides/compliance-forms/overview.md b/docs/features/feature-guides/compliance-forms/overview.md new file mode 100644 index 00000000000000..86a6d8cc6dadfb --- /dev/null +++ b/docs/features/feature-guides/compliance-forms/overview.md @@ -0,0 +1,46 @@ +--- +title: Overview +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# About DataHub Compliance Forms + + +**DataHub Compliance Forms** streamline the process of documenting, annotating, and classifying your most critical Data Assets through a collaborative, crowdsourced approach. + +With Compliance Forms, you can execute large-scale compliance initiatives by assigning tasks (e.g., documentation, tagging, or classification requirements) to the appropriate stakeholders — data owners, stewards, and subject matter experts. + +## What are Compliance Forms? + +A **Compliance Form** is a flexible and centrally managed tool that enables your data governance or compliance teams to define, enforce, and monitor requirements for specific Data Assets or Columns. + +A Compliance Form consists of: + +1. **Assets:** The Data Assets or Columns for which the Form must be completed. These represent the scope of the compliance initiative. + +2. **Questions:** The set of requirements or conditions that must be completed for each asset. Questions are a vehicle to collect key attributes for your data assets. These can range from simple to complex, with questions that require differing types of answers to complete. Examples include Descriptions, Domains, Owners, Tags, Glossary Terms, and custom Structured Properties. + +3. **Assignees:** The users or groups responsible for completing the Form (e.g., asset owners, domain experts, or stewards). + +Once a Compliance Form is defined, it can be published. When a form is published, the assignees who are required to complete the requirements will be notified via the Inbox of the tasks that they must complete. In addition, analytics will begin to be gathered about the assets that are meeting or violating the requirements in the form so you can understand your initiative's progress over time. + +### Why Use Compliance Forms? + +Compliance Forms enable organizations to: +- Standardize documentation and metadata across critical Data Assets. +- Crowdsource compliance-related tasks to domain experts who are best equipped to provide accurate information. +- Scale governance initiatives efficiently while maintaining accuracy and accountability. + +By leveraging Compliance Forms, organizations can ensure consistent metadata quality and foster collaboration between data experts and governance teams. + +

+ Sample Compliance Form +

+ +## Next Steps + +Now that you understand the basics of DataHub Compliance Forms, you're ready to [Create a Compliance Form](create-a-form.md). \ No newline at end of file diff --git a/docs/features/feature-guides/documentation-forms.md b/docs/features/feature-guides/documentation-forms.md deleted file mode 100644 index 2edeb8ce302d77..00000000000000 --- a/docs/features/feature-guides/documentation-forms.md +++ /dev/null @@ -1,113 +0,0 @@ -import FeatureAvailability from '@site/src/components/FeatureAvailability'; - -# About DataHub Documentation Forms - - -DataHub Documentation Forms streamline the process of setting documentation requirements and delegating annotation responsibilities to the relevant data asset owners, stewards, and subject matter experts. - -Forms are highly configurable, making it easy to ask the right questions of the right people, for a specific set of assets. - -## What are Documentation Forms? - -You can think of Documentation Forms as a survey for your data assets: a set of questions that must be answered in order for an asset to be considered properly documented. - -Verification Forms are an extension of Documentation Forms, requiring a final verification, or sign-off, on all responses before the asset can be considered Verified. This is useful for compliance and/or governance annotation initiatives where you want assignees to provide a final acknowledgement that the information provided is correct. - -## Creating and Assigning Documentation Forms - -Documentation Forms are defined via YAML with the following details: - -- Name and Description to help end-users understand the scope and use case -- Form Type, either Documentation or Verification - - Verification Forms require a final signoff, i.e. Verification, of all required questions before the Form can be considered complete -- Form Questions (aka "prompts") for end-users to complete - - Questions can be assigned at the asset-level and/or the field-level - - Asset-level questions can be configured to be required; by default, all questions are optional -- Assigned Assets, defined by: - - A set of specific asset URNs, OR - - Assets related to a set of filters, such as Type (Datasets, Dashboards, etc.), Platform (Snowflake, Looker, etc.), Domain (Product, Marketing, etc.), or Container (Schema, Folder, etc.) -- Optional: Form Assignees - - Optionally assign specific DataHub users/groups to complete the Form for all relevant assets - - If omitted, any Owner of an Asset can complete Forms assigned to that Asset - -Here's an example of defining a Documentation Form via YAML: -```yaml -- id: 123456 - # urn: "urn:li:form:123456" # optional if id is provided - type: VERIFICATION # Supported Types: DOCUMENTATION, VERIFICATION - name: "Metadata Initiative 2024" - description: "How we want to ensure the most important data assets in our organization have all of the most important and expected pieces of metadata filled out" - prompts: # Questions for Form assignees to complete - - id: "123" - title: "Data Retention Time" - description: "Apply Retention Time structured property to form" - type: STRUCTURED_PROPERTY - structured_property_id: io.acryl.privacy.retentionTime - required: True # optional; default value is False - entities: # Either pass a list of urns or a group of filters. This example shows a list of urns - urns: - - urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD) - # optionally assign the form to a specific set of users and/or groups - # when omitted, form will be assigned to Asset owners - actors: - users: - - urn:li:corpuser:jane@email.com # note: these should be URNs - - urn:li:corpuser:john@email.com - groups: - - urn:li:corpGroup:team@email.com # note: these should be URNs - -``` - -:::note -Documentation Forms currently only support defining Structured Properties as Form Questions -::: - - - - - -## Additional Resources - -### Videos - -**Asset Verification in DataHub Cloud** - -

- -

- -## FAQ and Troubleshooting - -**What is the difference between Documentation and Verification Forms?** - -Both form types are a way to configure a set of optional and/or required questions for DataHub users to complete. When using Verification Forms, users will be presented with a final verification step once all required questions have been completed; you can think of this as a final acknowledgement of the accuracy of information submitted. - -**Who is able to complete Forms in DataHub?** - -By default, any owner of an Asset will be able to respond to questions assigned via a Form. - -When assigning a Form to an Asset, you can optionally assign specific DataHub users/groups to fill them out. - -**Can I assign multiple Forms to a single asset?** - -You sure can! Please keep in mind that an Asset will only be considered Documented or Verified if all required questions are completed on all assiged Forms. - -### API Tutorials - -- [API Guides on Documentation Form](../../../docs/api/tutorials/forms.md) - -:::note -You must create a Structured Property before including it in a Documentation Form. -To learn more about creating Structured Properties via CLI, please see the [Create Structured Properties](/docs/api/tutorials/structured-properties.md) tutorial. -::: - -### Related Features - -- [DataHub Properties](/docs/features/feature-guides/properties.md) \ No newline at end of file diff --git a/docs/features/feature-guides/properties.md b/docs/features/feature-guides/properties.md index 0d961b9ceac4ff..abdb736ad2a429 100644 --- a/docs/features/feature-guides/properties.md +++ b/docs/features/feature-guides/properties.md @@ -155,4 +155,4 @@ Please see the following API guides related to Custom and Structured Properties: ### Related Features -- [Documentation Forms](/docs/features/feature-guides/documentation-forms.md) \ No newline at end of file +- [Compliance Forms](compliance-forms/overview.md) \ No newline at end of file From 1ed55f417606477071f7843be87caff0ff1ea4c8 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Thu, 5 Dec 2024 17:50:56 -0800 Subject: [PATCH 142/174] feat(snowflake): adding oauth token bypass to snowflake (#12048) --- .../source/snowflake/snowflake_connection.py | 28 ++++++++++ .../unit/snowflake/test_snowflake_source.py | 54 +++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py index 397606400d389c..2239338972d9be 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_connection.py @@ -43,6 +43,7 @@ "EXTERNAL_BROWSER_AUTHENTICATOR": EXTERNAL_BROWSER_AUTHENTICATOR, "KEY_PAIR_AUTHENTICATOR": KEY_PAIR_AUTHENTICATOR, "OAUTH_AUTHENTICATOR": OAUTH_AUTHENTICATOR, + "OAUTH_AUTHENTICATOR_TOKEN": OAUTH_AUTHENTICATOR, } _SNOWFLAKE_HOST_SUFFIX = ".snowflakecomputing.com" @@ -104,6 +105,10 @@ class SnowflakeConnectionConfig(ConfigModel): description="Connect args to pass to Snowflake SqlAlchemy driver", exclude=True, ) + token: Optional[str] = pydantic.Field( + default=None, + description="OAuth token from external identity provider. Not recommended for most use cases because it will not be able to refresh once expired.", + ) def get_account(self) -> str: assert self.account_id @@ -148,6 +153,18 @@ def authenticator_type_is_valid(cls, v, values): logger.info(f"using authenticator type '{v}'") return v + @pydantic.validator("token", always=True) + def validate_token_oauth_config(cls, v, values): + auth_type = values.get("authentication_type") + if auth_type == "OAUTH_AUTHENTICATOR_TOKEN": + if not v: + raise ValueError("Token required for OAUTH_AUTHENTICATOR_TOKEN.") + elif v is not None: + raise ValueError( + "Token can only be provided when using OAUTH_AUTHENTICATOR_TOKEN" + ) + return v + @staticmethod def _check_oauth_config(oauth_config: Optional[OAuthConfiguration]) -> None: if oauth_config is None: @@ -333,6 +350,17 @@ def get_native_connection(self) -> NativeSnowflakeConnection: application=_APPLICATION_NAME, **connect_args, ) + elif self.authentication_type == "OAUTH_AUTHENTICATOR_TOKEN": + return snowflake.connector.connect( + user=self.username, + account=self.account_id, + authenticator="oauth", + token=self.token, # Token generated externally and provided directly to the recipe + warehouse=self.warehouse, + role=self.role, + application=_APPLICATION_NAME, + **connect_args, + ) elif self.authentication_type == "OAUTH_AUTHENTICATOR": return self.get_oauth_connection() elif self.authentication_type == "KEY_PAIR_AUTHENTICATOR": diff --git a/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py index 161dfa2b4e78f3..3284baf103e5af 100644 --- a/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py +++ b/metadata-ingestion/tests/unit/snowflake/test_snowflake_source.py @@ -130,6 +130,60 @@ def test_snowflake_oauth_happy_paths(): ) +def test_snowflake_oauth_token_happy_path(): + assert SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "authentication_type": "OAUTH_AUTHENTICATOR_TOKEN", + "token": "valid-token", + "username": "test-user", + "oauth_config": None, + } + ) + + +def test_snowflake_oauth_token_without_token(): + with pytest.raises( + ValidationError, match="Token required for OAUTH_AUTHENTICATOR_TOKEN." + ): + SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "authentication_type": "OAUTH_AUTHENTICATOR_TOKEN", + "username": "test-user", + } + ) + + +def test_snowflake_oauth_token_with_wrong_auth_type(): + with pytest.raises( + ValueError, + match="Token can only be provided when using OAUTH_AUTHENTICATOR_TOKEN.", + ): + SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "authentication_type": "OAUTH_AUTHENTICATOR", + "token": "some-token", + "username": "test-user", + } + ) + + +def test_snowflake_oauth_token_with_empty_token(): + with pytest.raises( + ValidationError, match="Token required for OAUTH_AUTHENTICATOR_TOKEN." + ): + SnowflakeV2Config.parse_obj( + { + "account_id": "test", + "authentication_type": "OAUTH_AUTHENTICATOR_TOKEN", + "token": "", + "username": "test-user", + } + ) + + default_config_dict: Dict[str, Any] = { "username": "user", "password": "password", From 2fe21329fa34e8c3f9946870966756c64351c9f4 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:43:05 +0530 Subject: [PATCH 143/174] fix(ingest): avoid shell entities during view lineage generation (#12044) --- .../ingestion/source/sql/sql_common.py | 2 + .../ingestion/source/sql/sql_report.py | 1 + .../datahub/ingestion/source/unity/source.py | 2 + .../datahub/sql_parsing/sqlglot_lineage.py | 8 +- .../golden_test_ingest_with_database.json | 194 ++++++++---------- .../golden_mces_mssql_no_db_to_file.json | 54 +---- .../golden_mces_mssql_no_db_with_filter.json | 28 +-- .../golden_mces_mssql_to_file.json | 32 +-- 8 files changed, 118 insertions(+), 203 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 41ffcb95a7cc43..64aa8cfc6ef6c7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -1197,6 +1197,8 @@ def _run_sql_parser( ) else: self.report.num_view_definitions_parsed += 1 + if raw_lineage.out_tables != [view_urn]: + self.report.num_view_definitions_view_urn_mismatch += 1 return view_definition_lineage_helper(raw_lineage, view_urn) def get_db_schema(self, dataset_identifier: str) -> Tuple[Optional[str], str]: diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py index c1f722b5d1e783..c445ce44a91449 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_report.py @@ -48,6 +48,7 @@ class SQLSourceReport( query_combiner: Optional[SQLAlchemyQueryCombinerReport] = None num_view_definitions_parsed: int = 0 + num_view_definitions_view_urn_mismatch: int = 0 num_view_definitions_failed_parsing: int = 0 num_view_definitions_failed_column_parsing: int = 0 view_definitions_parsing_failures: LossyList[str] = field(default_factory=LossyList) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index 9a6cde78cf10d3..f758746193cd83 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -974,6 +974,8 @@ def _run_sql_parser( ) else: self.report.num_view_definitions_parsed += 1 + if raw_lineage.out_tables != [view_urn]: + self.report.num_view_definitions_view_urn_mismatch += 1 return view_definition_lineage_helper(raw_lineage, view_urn) def get_view_lineage(self) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py index 9adb792a4be518..4ff68574bf20e6 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py @@ -1243,13 +1243,19 @@ def infer_output_schema(result: SqlParsingResult) -> Optional[List[SchemaFieldCl def view_definition_lineage_helper( result: SqlParsingResult, view_urn: str ) -> SqlParsingResult: - if result.query_type is QueryType.SELECT: + if result.query_type is QueryType.SELECT or ( + result.out_tables and result.out_tables != [view_urn] + ): # Some platforms (e.g. postgres) store only ` . For such view definitions, `result.out_tables` and # `result.column_lineage[].downstream` are empty in `sqlglot_lineage` response, whereas upstream # details and downstream column details are extracted correctly. # Here, we inject view V's urn in `result.out_tables` and `result.column_lineage[].downstream` # to get complete lineage result. + + # Some platforms(e.g. mssql) may have slightly different view name in view definition than + # actual view name used elsewhere. Therefore we overwrite downstream table for such cases as well. + result.out_tables = [view_urn] if result.column_lineage: for col_result in result.column_lineage: diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json index cbcadde6feb213..abd9b2350638a2 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json @@ -17,7 +17,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -33,7 +33,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -67,7 +67,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -83,7 +83,23 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:937a38ee28b69ecae38665c5e842d0ad", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -106,7 +122,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -122,7 +138,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -138,7 +154,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -156,23 +172,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:937a38ee28b69ecae38665c5e842d0ad", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -193,7 +193,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -209,7 +209,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -272,7 +272,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -290,7 +290,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -315,7 +315,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -331,7 +331,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -394,7 +394,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -412,7 +412,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -437,7 +437,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -453,7 +453,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -519,7 +519,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -537,7 +537,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -555,7 +555,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -580,7 +580,23 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1965527855ae77f259a8ddea2b8eed2f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -603,7 +619,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -619,7 +635,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -635,7 +651,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -653,23 +669,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1965527855ae77f259a8ddea2b8eed2f", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:0e497517e191d344b0c403231bc708d0" - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -690,7 +690,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -706,7 +706,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -769,7 +769,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -787,7 +787,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -812,7 +812,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -828,7 +828,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -891,7 +891,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -909,7 +909,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -934,7 +934,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -950,7 +950,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -1016,7 +1016,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -1034,7 +1034,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -1052,7 +1052,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, @@ -1077,13 +1077,13 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -1106,7 +1106,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN1)" ], "confidenceScore": 1.0 }, @@ -1117,7 +1117,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD),MOCK_COLUMN2)" ], "confidenceScore": 1.0 } @@ -1126,13 +1126,13 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -1155,7 +1155,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD),MOCK_COLUMN1)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN1)" ], "confidenceScore": 1.0 }, @@ -1166,7 +1166,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD),MOCK_COLUMN2)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD),MOCK_COLUMN2)" ], "confidenceScore": 1.0 } @@ -1175,39 +1175,7 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema1.mock_view,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,oradoc.schema2.mock_view,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", + "runId": "oracle-2022_02_03-07_00_00-uzcdxn", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 54821347fd28b8..72dcda25c1296c 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", + "job_id": "c2d77890-83ba-435f-879b-1c77fa38dd47", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-22 12:58:03.260000", - "date_modified": "2024-11-22 12:58:03.440000", + "date_created": "2024-12-05 16:44:43.910000", + "date_modified": "2024-12-05 16:44:44.043000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2282,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-22 12:58:03.137000", - "date_modified": "2024-11-22 12:58:03.137000" + "date_created": "2024-12-05 16:44:43.800000", + "date_modified": "2024-12-05 16:44:43.800000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2310,8 +2310,8 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-11-22 12:58:03.140000", - "date_modified": "2024-11-22 12:58:03.140000" + "date_created": "2024-12-05 16:44:43.803000", + "date_modified": "2024-12-05 16:44:43.803000" }, "externalUrl": "", "name": "DemoData.Foo.NewProc", @@ -4883,7 +4883,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -4908,7 +4908,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -4931,7 +4931,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),firstname)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),firstname)" ], "confidenceScore": 1.0 }, @@ -4942,7 +4942,7 @@ ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD),lastname)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.View1,PROD),lastname)" ], "confidenceScore": 1.0 } @@ -5034,37 +5034,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,newdata.foonew.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 1d702214fedf79..0df89ff1eb94d7 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "2a055367-5e6a-4162-b3a9-dd60f52c79a8", + "job_id": "c2d77890-83ba-435f-879b-1c77fa38dd47", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-26 07:22:19.640000", - "date_modified": "2024-11-26 07:22:19.773000", + "date_created": "2024-12-05 16:44:43.910000", + "date_modified": "2024-12-05 16:44:44.043000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2282,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-26 07:22:19.510000", - "date_modified": "2024-11-26 07:22:19.510000" + "date_created": "2024-12-05 16:44:43.800000", + "date_modified": "2024-12-05 16:44:43.800000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2630,7 +2630,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -2716,21 +2716,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 3836e587ef8cf4..b67ebfb206883a 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -113,11 +113,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "4130c37d-146c-43da-a671-dd9a413a44b3", + "job_id": "c2d77890-83ba-435f-879b-1c77fa38dd47", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-11-22 12:58:03.260000", - "date_modified": "2024-11-22 12:58:03.440000", + "date_created": "2024-12-05 16:44:43.910000", + "date_modified": "2024-12-05 16:44:44.043000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -2282,8 +2282,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-11-22 12:58:03.137000", - "date_modified": "2024-11-22 12:58:03.137000" + "date_created": "2024-12-05 16:44:43.800000", + "date_modified": "2024-12-05 16:44:43.800000" }, "externalUrl": "", "name": "DemoData.Foo.Proc.With.SpecialChar", @@ -2310,8 +2310,8 @@ "depending_on_procedure": "{}", "code": "CREATE PROCEDURE [Foo].[NewProc]\n AS\n BEGIN\n --insert into items table from salesreason table\n insert into Foo.Items (ID, ItemName)\n SELECT TempID, Name\n FROM Foo.SalesReason;\n\n\n IF OBJECT_ID('Foo.age_dist', 'U') IS NULL\n BEGIN\n -- Create and populate if table doesn't exist\n SELECT Age, COUNT(*) as Count\n INTO Foo.age_dist\n FROM Foo.Persons\n GROUP BY Age\n END\n ELSE\n BEGIN\n -- Update existing table\n TRUNCATE TABLE Foo.age_dist;\n\n INSERT INTO Foo.age_dist (Age, Count)\n SELECT Age, COUNT(*) as Count\n FROM Foo.Persons\n GROUP BY Age\n END\n\n SELECT ID, Age INTO #TEMPTABLE FROM NewData.FooNew.PersonsNew\n \n UPDATE DemoData.Foo.Persons\n SET Age = t.Age\n FROM DemoData.Foo.Persons p\n JOIN #TEMPTABLE t ON p.ID = t.ID\n\n END\n", "input parameters": "[]", - "date_created": "2024-11-22 12:58:03.140000", - "date_modified": "2024-11-22 12:58:03.140000" + "date_created": "2024-12-05 16:44:43.803000", + "date_modified": "2024-12-05 16:44:43.803000" }, "externalUrl": "", "name": "DemoData.Foo.NewProc", @@ -2658,7 +2658,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.PersonsView,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -2760,21 +2760,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.personsview,PROD)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file From eef2077a55c45ff5024f6b28a35a682909a44c80 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Fri, 6 Dec 2024 16:57:07 +0530 Subject: [PATCH 144/174] fix(logs): add actor urn on unauthorised (#12030) --- .../authentication/filter/AuthenticationFilter.java | 11 +++++++---- .../auth/authentication/AuthServiceController.java | 6 ++++-- .../metadata/resources/entity/AspectResource.java | 6 +++--- .../metadata/resources/entity/EntityResource.java | 8 ++++---- .../linkedin/metadata/resources/usage/UsageStats.java | 5 +++-- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java index 0a54677eb6149b..30f98180f80180 100644 --- a/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java +++ b/metadata-service/auth-filter/src/main/java/com/datahub/auth/authentication/filter/AuthenticationFilter.java @@ -98,11 +98,12 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha } if (authentication != null) { + String actorUrnStr = authentication.getActor().toUrnStr(); // Successfully authenticated. log.debug( - String.format( - "Successfully authenticated request for Actor with type: %s, id: %s", - authentication.getActor().getType(), authentication.getActor().getId())); + "Successfully authenticated request for Actor with type: {}, id: {}", + authentication.getActor().getType(), + authentication.getActor().getId()); AuthenticationContext.setAuthentication(authentication); chain.doFilter(request, response); } else { @@ -110,7 +111,9 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha log.debug( "Failed to authenticate request. Received 'null' Authentication value from authenticator chain."); ((HttpServletResponse) response) - .sendError(HttpServletResponse.SC_UNAUTHORIZED, "Unauthorized to perform this action."); + .sendError( + HttpServletResponse.SC_UNAUTHORIZED, + "Unauthorized to perform this action due to expired auth."); return; } AuthenticationContext.remove(); diff --git a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java index de2582af00a932..5d4542cf0826e8 100644 --- a/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java +++ b/metadata-service/auth-servlet-impl/src/main/java/com/datahub/auth/authentication/AuthServiceController.java @@ -138,7 +138,9 @@ CompletableFuture> generateSessionTokenForUser( } log.info("Attempting to generate session token for user {}", userId.asText()); - final String actorId = AuthenticationContext.getAuthentication().getActor().getId(); + Authentication authentication = AuthenticationContext.getAuthentication(); + final String actorId = authentication.getActor().getId(); + final String actorUrn = authentication.getActor().toUrnStr(); return CompletableFuture.supplyAsync( () -> { // 1. Verify that only those authorized to generate a token (datahub system) are able to. @@ -164,7 +166,7 @@ CompletableFuture> generateSessionTokenForUser( } throw HttpClientErrorException.create( HttpStatus.UNAUTHORIZED, - "Unauthorized to perform this action.", + actorUrn + " unauthorized to perform this action.", new HttpHeaders(), null, null); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java index a8b9c34ab66ae6..6033ead36f10ec 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/AspectResource.java @@ -281,12 +281,13 @@ private Task ingestProposals( boolean asyncBool) throws URISyntaxException { Authentication authentication = AuthenticationContext.getAuthentication(); + String actorUrnStr = authentication.getActor().toUrnStr(); Set entityTypes = metadataChangeProposals.stream() .map(MetadataChangeProposal::getEntityType) .collect(Collectors.toSet()); final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(), + systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr, getContext(), ACTION_INGEST_PROPOSAL, entityTypes), _authorizer, authentication, true); // Ingest Authorization Checks @@ -299,9 +300,8 @@ private Task ingestProposals( .map(ex -> String.format("HttpStatus: %s Urn: %s", ex.getSecond(), ex.getFirst().getEntityUrn())) .collect(Collectors.joining(", ")); throw new RestLiServiceException( - HttpStatus.S_403_FORBIDDEN, "User is unauthorized to modify entity: " + errorMessages); + HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr + " is unauthorized to modify entity: " + errorMessages); } - String actorUrnStr = authentication.getActor().toUrnStr(); final AuditStamp auditStamp = new AuditStamp().setTime(_clock.millis()).setActor(Urn.createFromString(actorUrnStr)); diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 6c5576f2e5d9f4..0c374c29cf958a 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -274,7 +274,7 @@ public Task ingest( String actorUrnStr = authentication.getActor().toUrnStr(); final Urn urn = com.datahub.util.ModelUtils.getUrnFromSnapshotUnion(entity.getValue()); final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), getContext(), + systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr, getContext(), ACTION_INGEST, urn.getEntityType()), authorizer, authentication, true); if (!isAPIAuthorizedEntityUrns( @@ -282,7 +282,7 @@ public Task ingest( CREATE, List.of(urn))) { throw new RestLiServiceException( - HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entity " + urn); + HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr + " is unauthorized to edit entity " + urn); } try { @@ -320,7 +320,7 @@ public Task batchIngest( .map(Entity::getValue) .map(com.datahub.util.ModelUtils::getUrnFromSnapshotUnion).collect(Collectors.toList()); final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(authentication.getActor().toUrnStr(), + systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr, getContext(), ACTION_BATCH_INGEST, urns.stream().map(Urn::getEntityType).collect(Collectors.toList())), authorizer, authentication, true); @@ -328,7 +328,7 @@ public Task batchIngest( opContext, CREATE, urns)) { throw new RestLiServiceException( - HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entities."); + HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr + " is unauthorized to edit entities."); } for (Entity entity : entities) { diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java index a0c3d460951605..426eff20c9c6eb 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java @@ -104,9 +104,10 @@ public Task batchIngest(@ActionParam(PARAM_BUCKETS) @Nonnull UsageAggregat () -> { final Authentication auth = AuthenticationContext.getAuthentication(); + String actorUrnStr = auth.getActor().toUrnStr(); Set urns = Arrays.stream(buckets).sequential().map(UsageAggregation::getResource).collect(Collectors.toSet()); final OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(auth.getActor().toUrnStr(), getContext(), + systemOperationContext, RequestContext.builder().buildRestli(actorUrnStr, getContext(), ACTION_BATCH_INGEST, urns.stream().map(Urn::getEntityType).collect(Collectors.toList())), _authorizer, auth, true); @@ -115,7 +116,7 @@ public Task batchIngest(@ActionParam(PARAM_BUCKETS) @Nonnull UsageAggregat UPDATE, urns)) { throw new RestLiServiceException( - HttpStatus.S_403_FORBIDDEN, "User is unauthorized to edit entities."); + HttpStatus.S_403_FORBIDDEN, "User " + actorUrnStr + " is unauthorized to edit entities."); } for (UsageAggregation agg : buckets) { From ea9eaf439d93999d6ceb2e13aff46a4a2adae790 Mon Sep 17 00:00:00 2001 From: stcha <100420854+siong-tcha@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:02:32 +0100 Subject: [PATCH 145/174] fix(ingest/snowflake): Add handling of Hybrid Table type for Snowflake ingestion (#12039) --- .../src/datahub/ingestion/source/snowflake/snowflake_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 662e1cc2509eae..bb5d0636f67123 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -132,7 +132,7 @@ def tables_for_database(db_name: Optional[str]) -> str: auto_clustering_on AS "AUTO_CLUSTERING_ON" FROM {db_clause}information_schema.tables t WHERE table_schema != 'INFORMATION_SCHEMA' - and table_type in ( 'BASE TABLE', 'EXTERNAL TABLE') + and table_type in ( 'BASE TABLE', 'EXTERNAL TABLE', 'HYBRID TABLE') order by table_schema, table_name""" @staticmethod @@ -152,7 +152,7 @@ def tables_for_schema(schema_name: str, db_name: Optional[str]) -> str: auto_clustering_on AS "AUTO_CLUSTERING_ON" FROM {db_clause}information_schema.tables t where table_schema='{schema_name}' - and table_type in ('BASE TABLE', 'EXTERNAL TABLE') + and table_type in ('BASE TABLE', 'EXTERNAL TABLE', 'HYBRID TABLE') order by table_schema, table_name""" @staticmethod From b495205ed03d425093d509263e76c1b7c3f467d3 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 6 Dec 2024 11:13:37 -0500 Subject: [PATCH 146/174] fix(ingest/powerbi): reduce type cast usage (#12004) --- .../source/powerbi/m_query/data_classes.py | 15 +----- .../source/powerbi/m_query/pattern_handler.py | 46 ++++++++----------- .../source/powerbi/m_query/resolver.py | 18 ++++---- .../source/powerbi/m_query/tree_function.py | 6 +-- .../source/snowflake/snowflake_config.py | 7 +-- 5 files changed, 36 insertions(+), 56 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py index f1691b5df68a94..8c9ba3b458ab25 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/data_classes.py @@ -1,5 +1,4 @@ import os -from abc import ABC from dataclasses import dataclass from enum import Enum from typing import Any, Dict, List, Optional @@ -12,18 +11,8 @@ TRACE_POWERBI_MQUERY_PARSER = os.getenv("DATAHUB_TRACE_POWERBI_MQUERY_PARSER", False) -class AbstractIdentifierAccessor(ABC): # To pass lint - pass - - -# @dataclass -# class ItemSelector: -# items: Dict[str, Any] -# next: Optional[AbstractIdentifierAccessor] - - @dataclass -class IdentifierAccessor(AbstractIdentifierAccessor): +class IdentifierAccessor: """ statement public_order_date = Source{[Schema="public",Item="order_date"]}[Data] @@ -40,7 +29,7 @@ class IdentifierAccessor(AbstractIdentifierAccessor): identifier: str items: Dict[str, Any] - next: Optional[AbstractIdentifierAccessor] + next: Optional["IdentifierAccessor"] @dataclass diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py index 13d97a70290298..ffaed79f4e42a6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/pattern_handler.py @@ -1,7 +1,7 @@ import logging from abc import ABC, abstractmethod from enum import Enum -from typing import Dict, List, Optional, Tuple, Type, Union, cast +from typing import Dict, List, Optional, Tuple, Type, cast from lark import Tree @@ -22,7 +22,6 @@ ) from datahub.ingestion.source.powerbi.m_query import native_sql_parser, tree_function from datahub.ingestion.source.powerbi.m_query.data_classes import ( - AbstractIdentifierAccessor, DataAccessFunctionDetail, DataPlatformTable, FunctionName, @@ -412,33 +411,25 @@ def create_lineage( ) table_detail: Dict[str, str] = {} temp_accessor: Optional[ - Union[IdentifierAccessor, AbstractIdentifierAccessor] + IdentifierAccessor ] = data_access_func_detail.identifier_accessor while temp_accessor: - if isinstance(temp_accessor, IdentifierAccessor): - # Condition to handle databricks M-query pattern where table, schema and database all are present in - # the same invoke statement - if all( - element in temp_accessor.items - for element in ["Item", "Schema", "Catalog"] - ): - table_detail["Schema"] = temp_accessor.items["Schema"] - table_detail["Table"] = temp_accessor.items["Item"] - else: - table_detail[temp_accessor.items["Kind"]] = temp_accessor.items[ - "Name" - ] - - if temp_accessor.next is not None: - temp_accessor = temp_accessor.next - else: - break + # Condition to handle databricks M-query pattern where table, schema and database all are present in + # the same invoke statement + if all( + element in temp_accessor.items + for element in ["Item", "Schema", "Catalog"] + ): + table_detail["Schema"] = temp_accessor.items["Schema"] + table_detail["Table"] = temp_accessor.items["Item"] else: - logger.debug( - "expecting instance to be IdentifierAccessor, please check if parsing is done properly" - ) - return Lineage.empty() + table_detail[temp_accessor.items["Kind"]] = temp_accessor.items["Name"] + + if temp_accessor.next is not None: + temp_accessor = temp_accessor.next + else: + break table_reference = self.create_reference_table( arg_list=data_access_func_detail.arg_list, @@ -786,9 +777,10 @@ def get_db_name(self, data_access_tokens: List[str]) -> Optional[str]: def create_lineage( self, data_access_func_detail: DataAccessFunctionDetail ) -> Lineage: - t1: Tree = cast( - Tree, tree_function.first_arg_list_func(data_access_func_detail.arg_list) + t1: Optional[Tree] = tree_function.first_arg_list_func( + data_access_func_detail.arg_list ) + assert t1 is not None flat_argument_list: List[Tree] = tree_function.flat_argument_list(t1) if len(flat_argument_list) != 2: diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py index 81a0e1ef2d79b1..2756a113d1ef0c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/resolver.py @@ -1,6 +1,6 @@ import logging from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Dict, List, Optional, Tuple, Union from lark import Tree @@ -95,14 +95,12 @@ def get_item_selector_tokens( # remove whitespaces and quotes from token tokens: List[str] = tree_function.strip_char_from_list( tree_function.remove_whitespaces_from_list( - tree_function.token_values( - cast(Tree, item_selector), parameters=self.parameters - ) + tree_function.token_values(item_selector, parameters=self.parameters) ), ) identifier: List[str] = tree_function.token_values( - cast(Tree, identifier_tree) - ) # type :ignore + identifier_tree, parameters={} + ) # convert tokens to dict iterator = iter(tokens) @@ -238,10 +236,10 @@ def _process_invoke_expression( def _process_item_selector_expression( self, rh_tree: Tree ) -> Tuple[Optional[str], Optional[Dict[str, str]]]: - new_identifier, key_vs_value = self.get_item_selector_tokens( # type: ignore - cast(Tree, tree_function.first_expression_func(rh_tree)) - ) + first_expression: Optional[Tree] = tree_function.first_expression_func(rh_tree) + assert first_expression is not None + new_identifier, key_vs_value = self.get_item_selector_tokens(first_expression) return new_identifier, key_vs_value @staticmethod @@ -327,7 +325,7 @@ def internal( # The first argument can be a single table argument or list of table. # For example Table.Combine({t1,t2},....), here first argument is list of table. # Table.AddColumn(t1,....), here first argument is single table. - for token in cast(List[str], result): + for token in result: internal(token, identifier_accessor) else: diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py index 186f03fe136393..d48e251bd00906 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/m_query/tree_function.py @@ -1,6 +1,6 @@ import logging from functools import partial -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, Dict, List, Optional, Union from lark import Token, Tree @@ -58,7 +58,7 @@ def internal(node: Union[Tree, Token]) -> Optional[Tree]: if isinstance(node, Token): return None - for child in cast(Tree, node).children: + for child in node.children: child_node: Optional[Tree] = internal(child) if child_node is not None: return child_node @@ -99,7 +99,7 @@ def internal(node: Union[Tree, Token]) -> None: logger.debug(f"Unable to resolve parameter reference to {ref}") values.append(ref) elif isinstance(node, Token): - values.append(cast(Token, node).value) + values.append(node.value) return else: for child in node.children: diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 229c0e292fbafe..c30a26fbbd02cc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -1,7 +1,7 @@ import logging from collections import defaultdict from dataclasses import dataclass -from typing import Dict, List, Optional, Set, cast +from typing import Dict, List, Optional, Set import pydantic from pydantic import Field, SecretStr, root_validator, validator @@ -118,9 +118,10 @@ def validate_legacy_schema_pattern(cls, values: Dict) -> Dict: ) # Always exclude reporting metadata for INFORMATION_SCHEMA schema - if schema_pattern is not None and schema_pattern: + if schema_pattern: logger.debug("Adding deny for INFORMATION_SCHEMA to schema_pattern.") - cast(AllowDenyPattern, schema_pattern).deny.append(r".*INFORMATION_SCHEMA$") + assert isinstance(schema_pattern, AllowDenyPattern) + schema_pattern.deny.append(r".*INFORMATION_SCHEMA$") return values From bd7649ed89c15dfccceb7f3b1ed013db6c478211 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 6 Dec 2024 11:17:41 -0500 Subject: [PATCH 147/174] refactor(ingest/sql): add _get_view_definition helper method (#12033) --- .../ingestion/source/sql/mssql/source.py | 2 - .../ingestion/source/sql/sql_common.py | 53 +++++++++++-------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py index 7a2dbda8b4a939..414c1faaa1661a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/mssql/source.py @@ -5,8 +5,6 @@ import pydantic import sqlalchemy.dialects.mssql - -# This import verifies that the dependencies are available. from pydantic.fields import Field from sqlalchemy import create_engine, inspect from sqlalchemy.engine.base import Connection diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index 64aa8cfc6ef6c7..4e22930e7a2a0b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -582,6 +582,8 @@ def get_view_lineage(self) -> Iterable[MetadataWorkUnit]: generate_operations=False, ) for dataset_name in self._view_definition_cache.keys(): + # TODO: Ensure that the lineage generated from the view definition + # matches the dataset_name. view_definition = self._view_definition_cache[dataset_name] result = self._run_sql_parser( dataset_name, @@ -1059,6 +1061,20 @@ def loop_views( exc=e, ) + def _get_view_definition(self, inspector: Inspector, schema: str, view: str) -> str: + try: + view_definition = inspector.get_view_definition(view, schema) + if view_definition is None: + view_definition = "" + else: + # Some dialects return a TextClause instead of a raw string, + # so we need to convert them to a string. + view_definition = str(view_definition) + except NotImplementedError: + view_definition = "" + + return view_definition + def _process_view( self, dataset_name: str, @@ -1077,7 +1093,10 @@ def _process_view( columns = inspector.get_columns(view, schema) except KeyError: # For certain types of views, we are unable to fetch the list of columns. - self.warn(logger, dataset_name, "unable to get schema for this view") + self.report.warning( + message="Unable to get schema for a view", + context=f"{dataset_name}", + ) schema_metadata = None else: schema_fields = self.get_schema_fields(dataset_name, columns, inspector) @@ -1091,19 +1110,12 @@ def _process_view( if self._save_schema_to_resolver(): self.schema_resolver.add_schema_metadata(dataset_urn, schema_metadata) self.discovered_datasets.add(dataset_name) + description, properties, _ = self.get_table_properties(inspector, schema, view) - try: - view_definition = inspector.get_view_definition(view, schema) - if view_definition is None: - view_definition = "" - else: - # Some dialects return a TextClause instead of a raw string, - # so we need to convert them to a string. - view_definition = str(view_definition) - except NotImplementedError: - view_definition = "" - properties["view_definition"] = view_definition properties["is_view"] = "True" + + view_definition = self._get_view_definition(inspector, schema, view) + properties["view_definition"] = view_definition if view_definition and self.config.include_view_lineage: self._view_definition_cache[dataset_name] = view_definition @@ -1135,15 +1147,14 @@ def _process_view( entityUrn=dataset_urn, aspect=SubTypesClass(typeNames=[DatasetSubTypes.VIEW]), ).as_workunit() - if "view_definition" in properties: - view_definition_string = properties["view_definition"] - view_properties_aspect = ViewPropertiesClass( - materialized=False, viewLanguage="SQL", viewLogic=view_definition_string - ) - yield MetadataChangeProposalWrapper( - entityUrn=dataset_urn, - aspect=view_properties_aspect, - ).as_workunit() + + view_properties_aspect = ViewPropertiesClass( + materialized=False, viewLanguage="SQL", viewLogic=view_definition + ) + yield MetadataChangeProposalWrapper( + entityUrn=dataset_urn, + aspect=view_properties_aspect, + ).as_workunit() if self.config.domain and self.domain_registry: yield from get_domain_wu( From 46aa962bad67e0523b2707e1d535f6bf53c227f0 Mon Sep 17 00:00:00 2001 From: hwmarkcheng <94201005+hwmarkcheng@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:48:00 -0500 Subject: [PATCH 148/174] feat(ingest/superset): initial support for superset datasets (#11972) --- .../src/datahub/ingestion/source/preset.py | 1 + .../src/datahub/ingestion/source/superset.py | 280 ++++++++--- .../superset/golden_test_ingest.json | 2 + .../superset/golden_test_stateful_ingest.json | 476 ++++++++++++++++-- .../integration/superset/test_superset.py | 343 ++++++++++++- .../tests/unit/test_preset_source.py | 20 + 6 files changed, 1020 insertions(+), 102 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/preset.py b/metadata-ingestion/src/datahub/ingestion/source/preset.py index 6f53223e000f1b..7b0bc89648c529 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/preset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/preset.py @@ -85,6 +85,7 @@ def __init__(self, ctx: PipelineContext, config: PresetConfig): super().__init__(ctx, config) self.config = config self.report = StaleEntityRemovalSourceReport() + self.platform = "preset" def login(self): try: diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 5ce33da5c55fac..1da233bf0b22ab 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -1,10 +1,12 @@ import json import logging +from datetime import datetime from functools import lru_cache -from typing import Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional import dateutil.parser as dp import requests +from pydantic import BaseModel from pydantic.class_validators import root_validator, validator from pydantic.fields import Field @@ -16,7 +18,9 @@ from datahub.emitter.mce_builder import ( make_chart_urn, make_dashboard_urn, + make_data_platform_urn, make_dataset_urn, + make_dataset_urn_with_platform_instance, make_domain_urn, ) from datahub.emitter.mcp_builder import add_domain_to_entity_wu @@ -31,6 +35,7 @@ ) from datahub.ingestion.api.source import MetadataWorkUnitProcessor, Source from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.sql.sql_types import resolve_sql_type from datahub.ingestion.source.sql.sqlalchemy_uri_mapper import ( get_platform_from_sqlalchemy_uri, ) @@ -47,16 +52,26 @@ AuditStamp, ChangeAuditStamps, Status, + TimeStamp, ) from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import ( ChartSnapshot, DashboardSnapshot, + DatasetSnapshot, ) from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + MySqlDDL, + NullType, + SchemaField, + SchemaFieldDataType, + SchemaMetadata, +) from datahub.metadata.schema_classes import ( ChartInfoClass, ChartTypeClass, DashboardInfoClass, + DatasetPropertiesClass, ) from datahub.utilities import config_clean from datahub.utilities.registries.domain_registry import DomainRegistry @@ -82,9 +97,29 @@ "box_plot": ChartTypeClass.BAR, } + platform_without_databases = ["druid"] +class SupersetDataset(BaseModel): + id: int + table_name: str + changed_on_utc: Optional[str] = None + explore_url: Optional[str] = "" + + @property + def modified_dt(self) -> Optional[datetime]: + if self.changed_on_utc: + return dp.parse(self.changed_on_utc) + return None + + @property + def modified_ts(self) -> Optional[int]: + if self.modified_dt: + return int(self.modified_dt.timestamp() * 1000) + return None + + class SupersetConfig( StatefulIngestionConfigBase, EnvConfigMixin, PlatformInstanceConfigMixin ): @@ -103,15 +138,17 @@ class SupersetConfig( ) username: Optional[str] = Field(default=None, description="Superset username.") password: Optional[str] = Field(default=None, description="Superset password.") - api_key: Optional[str] = Field(default=None, description="Preset.io API key.") - api_secret: Optional[str] = Field(default=None, description="Preset.io API secret.") - manager_uri: str = Field( - default="https://api.app.preset.io", description="Preset.io API URL" - ) # Configuration for stateful ingestion stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field( default=None, description="Superset Stateful Ingestion Config." ) + ingest_dashboards: bool = Field( + default=True, description="Enable to ingest dashboards." + ) + ingest_charts: bool = Field(default=True, description="Enable to ingest charts.") + ingest_datasets: bool = Field( + default=False, description="Enable to ingest datasets." + ) provider: str = Field(default="db", description="Superset provider.") options: Dict = Field(default={}, description="") @@ -123,6 +160,10 @@ class SupersetConfig( description="Can be used to change mapping for database names in superset to what you have in datahub", ) + class Config: + # This is required to allow preset configs to get parsed + extra = "allow" + @validator("connect_uri", "display_uri") def remove_trailing_slash(cls, v): return config_clean.remove_trailing_slashes(v) @@ -229,6 +270,28 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source: config = SupersetConfig.parse_obj(config_dict) return cls(ctx, config) + def paginate_entity_api_results(self, entity_type, page_size=100): + current_page = 0 + total_items = page_size + + while current_page * page_size < total_items: + response = self.session.get( + f"{self.config.connect_uri}/api/v1/{entity_type}/", + params={"q": f"(page:{current_page},page_size:{page_size})"}, + ) + + if response.status_code != 200: + logger.warning(f"Failed to get {entity_type} data: {response.text}") + + payload = response.json() + # Update total_items with the actual count from the response + total_items = payload.get("count", total_items) + # Yield each item in the result, this gets passed into the construct functions + for item in payload.get("result", []): + yield item + + current_page += 1 + @lru_cache(maxsize=None) def get_platform_from_database_id(self, database_id): database_response = self.session.get( @@ -250,11 +313,18 @@ def get_platform_from_database_id(self, database_id): return platform_name @lru_cache(maxsize=None) - def get_datasource_urn_from_id(self, datasource_id): + def get_dataset_info(self, dataset_id: int) -> dict: dataset_response = self.session.get( - f"{self.config.connect_uri}/api/v1/dataset/{datasource_id}" - ).json() - + f"{self.config.connect_uri}/api/v1/dataset/{dataset_id}", + ) + if dataset_response.status_code != 200: + logger.warning(f"Failed to get dataset info: {dataset_response.text}") + dataset_response.raise_for_status() + return dataset_response.json() + + def get_datasource_urn_from_id( + self, dataset_response: dict, platform_instance: str + ) -> str: schema_name = dataset_response.get("result", {}).get("schema") table_name = dataset_response.get("result", {}).get("table_name") database_id = dataset_response.get("result", {}).get("database", {}).get("id") @@ -283,9 +353,11 @@ def get_datasource_urn_from_id(self, datasource_id): ), env=self.config.env, ) - return None + raise ValueError("Could not construct dataset URN") - def construct_dashboard_from_api_data(self, dashboard_data): + def construct_dashboard_from_api_data( + self, dashboard_data: dict + ) -> DashboardSnapshot: dashboard_urn = make_dashboard_urn( platform=self.platform, name=dashboard_data["id"], @@ -340,7 +412,7 @@ def construct_dashboard_from_api_data(self, dashboard_data): } if dashboard_data.get("certified_by"): - custom_properties["CertifiedBy"] = dashboard_data.get("certified_by") + custom_properties["CertifiedBy"] = dashboard_data.get("certified_by", "") custom_properties["CertificationDetails"] = str( dashboard_data.get("certification_details") ) @@ -358,38 +430,25 @@ def construct_dashboard_from_api_data(self, dashboard_data): return dashboard_snapshot def emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]: - current_dashboard_page = 0 - # we will set total dashboards to the actual number after we get the response - total_dashboards = PAGE_SIZE - - while current_dashboard_page * PAGE_SIZE <= total_dashboards: - dashboard_response = self.session.get( - f"{self.config.connect_uri}/api/v1/dashboard/", - params=f"q=(page:{current_dashboard_page},page_size:{PAGE_SIZE})", - ) - if dashboard_response.status_code != 200: - logger.warning( - f"Failed to get dashboard data: {dashboard_response.text}" - ) - dashboard_response.raise_for_status() - - payload = dashboard_response.json() - total_dashboards = payload.get("count") or 0 - - current_dashboard_page += 1 - - for dashboard_data in payload["result"]: + for dashboard_data in self.paginate_entity_api_results("dashboard", PAGE_SIZE): + try: dashboard_snapshot = self.construct_dashboard_from_api_data( dashboard_data ) - mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot) - yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce) - yield from self._get_domain_wu( - title=dashboard_data.get("dashboard_title", ""), - entity_urn=dashboard_snapshot.urn, + except Exception as e: + self.report.warning( + f"Failed to construct dashboard snapshot. Dashboard name: {dashboard_data.get('dashboard_title')}. Error: \n{e}" ) + continue + # Emit the dashboard + mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot) + yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce) + yield from self._get_domain_wu( + title=dashboard_data.get("dashboard_title", ""), + entity_urn=dashboard_snapshot.urn, + ) - def construct_chart_from_chart_data(self, chart_data): + def construct_chart_from_chart_data(self, chart_data: dict) -> ChartSnapshot: chart_urn = make_chart_urn( platform=self.platform, name=chart_data["id"], @@ -415,9 +474,12 @@ def construct_chart_from_chart_data(self, chart_data): chart_url = f"{self.config.display_uri}{chart_data.get('url', '')}" datasource_id = chart_data.get("datasource_id") - datasource_urn = self.get_datasource_urn_from_id(datasource_id) + dataset_response = self.get_dataset_info(datasource_id) + datasource_urn = self.get_datasource_urn_from_id( + dataset_response, self.platform + ) - params = json.loads(chart_data.get("params")) + params = json.loads(chart_data.get("params", "{}")) metrics = [ get_metric_name(metric) for metric in (params.get("metrics", []) or [params.get("metric")]) @@ -467,36 +529,124 @@ def construct_chart_from_chart_data(self, chart_data): return chart_snapshot def emit_chart_mces(self) -> Iterable[MetadataWorkUnit]: - current_chart_page = 0 - # we will set total charts to the actual number after we get the response - total_charts = PAGE_SIZE - - while current_chart_page * PAGE_SIZE <= total_charts: - chart_response = self.session.get( - f"{self.config.connect_uri}/api/v1/chart/", - params=f"q=(page:{current_chart_page},page_size:{PAGE_SIZE})", + for chart_data in self.paginate_entity_api_results("chart", PAGE_SIZE): + try: + chart_snapshot = self.construct_chart_from_chart_data(chart_data) + + mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot) + except Exception as e: + self.report.warning( + f"Failed to construct chart snapshot. Chart name: {chart_data.get('table_name')}. Error: \n{e}" + ) + continue + # Emit the chart + yield MetadataWorkUnit(id=chart_snapshot.urn, mce=mce) + yield from self._get_domain_wu( + title=chart_data.get("slice_name", ""), + entity_urn=chart_snapshot.urn, ) - if chart_response.status_code != 200: - logger.warning(f"Failed to get chart data: {chart_response.text}") - chart_response.raise_for_status() - current_chart_page += 1 + def gen_schema_fields(self, column_data: List[Dict[str, str]]) -> List[SchemaField]: + schema_fields: List[SchemaField] = [] + for col in column_data: + col_type = (col.get("type") or "").lower() + data_type = resolve_sql_type(col_type) + if data_type is None: + data_type = NullType() + + field = SchemaField( + fieldPath=col.get("column_name", ""), + type=SchemaFieldDataType(data_type), + nativeDataType="", + description=col.get("column_name", ""), + nullable=True, + ) + schema_fields.append(field) + return schema_fields + + def gen_schema_metadata( + self, + dataset_response: dict, + ) -> SchemaMetadata: + dataset_response = dataset_response.get("result", {}) + column_data = dataset_response.get("columns", []) + schema_metadata = SchemaMetadata( + schemaName=dataset_response.get("table_name", ""), + platform=make_data_platform_urn(self.platform), + version=0, + hash="", + platformSchema=MySqlDDL(tableSchema=""), + fields=self.gen_schema_fields(column_data), + ) + return schema_metadata - payload = chart_response.json() - total_charts = payload["count"] - for chart_data in payload["result"]: - chart_snapshot = self.construct_chart_from_chart_data(chart_data) + def gen_dataset_urn(self, datahub_dataset_name: str) -> str: + return make_dataset_urn_with_platform_instance( + platform=self.platform, + name=datahub_dataset_name, + platform_instance=self.config.platform_instance, + env=self.config.env, + ) - mce = MetadataChangeEvent(proposedSnapshot=chart_snapshot) - yield MetadataWorkUnit(id=chart_snapshot.urn, mce=mce) - yield from self._get_domain_wu( - title=chart_data.get("slice_name", ""), - entity_urn=chart_snapshot.urn, + def construct_dataset_from_dataset_data( + self, dataset_data: dict + ) -> DatasetSnapshot: + dataset_response = self.get_dataset_info(dataset_data.get("id")) + dataset = SupersetDataset(**dataset_response["result"]) + datasource_urn = self.get_datasource_urn_from_id( + dataset_response, self.platform + ) + + dataset_url = f"{self.config.display_uri}{dataset.explore_url or ''}" + + dataset_info = DatasetPropertiesClass( + name=dataset.table_name, + description="", + lastModified=TimeStamp(time=dataset.modified_ts) + if dataset.modified_ts + else None, + externalUrl=dataset_url, + ) + aspects_items: List[Any] = [] + aspects_items.extend( + [ + self.gen_schema_metadata(dataset_response), + dataset_info, + ] + ) + + dataset_snapshot = DatasetSnapshot( + urn=datasource_urn, + aspects=aspects_items, + ) + return dataset_snapshot + + def emit_dataset_mces(self) -> Iterable[MetadataWorkUnit]: + for dataset_data in self.paginate_entity_api_results("dataset", PAGE_SIZE): + try: + dataset_snapshot = self.construct_dataset_from_dataset_data( + dataset_data ) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + except Exception as e: + self.report.warning( + f"Failed to construct dataset snapshot. Dataset name: {dataset_data.get('table_name')}. Error: \n{e}" + ) + continue + # Emit the dataset + yield MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce) + yield from self._get_domain_wu( + title=dataset_data.get("table_name", ""), + entity_urn=dataset_snapshot.urn, + ) def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: - yield from self.emit_dashboard_mces() - yield from self.emit_chart_mces() + if self.config.ingest_dashboards: + yield from self.emit_dashboard_mces() + if self.config.ingest_charts: + yield from self.emit_chart_mces() + if self.config.ingest_datasets: + yield from self.emit_dataset_mces() def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: return [ diff --git a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json index 767b85a72b975a..4801af9465f2c9 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json @@ -26,6 +26,7 @@ "urn:li:chart:(superset,11)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -73,6 +74,7 @@ "urn:li:chart:(superset,13)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, diff --git a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json index 0f207990179793..ac6a3b6942a32f 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json @@ -26,6 +26,7 @@ "urn:li:chart:(superset,11)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -44,7 +45,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00", + "runId": "superset-2020_04_14-07_00_00-83v7ts", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -92,7 +93,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00", + "runId": "superset-2020_04_14-07_00_00-83v7ts", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -140,7 +141,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00", + "runId": "superset-2020_04_14-07_00_00-83v7ts", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -188,47 +189,413 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00", + "runId": "superset-2020_04_14-07_00_00-83v7ts", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } }, { "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { - "urn": "urn:li:chart:(superset,13)", + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", "aspects": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] } }, { - "com.linkedin.pegasus2avro.chart.ChartInfo": { - "customProperties": { - "Metrics": "", - "Filters": "", - "Dimensions": "" + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, - "title": "test_chart_title_4", + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 1586847600000, - "actor": "urn:li:corpuser:test_username_2" + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" } }, - "chartUrl": "mock://mock-domain.superset.com/explore/test_chart_url_13", - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:external,test_database_name.test_schema_name.test_table_name,PROD)" + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" } - ], - "type": "HISTOGRAM" + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "Test Table 2", + "platform": "urn:li:dataPlatform:superset", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "externalUrl": "mock://mock-domain.superset.com", + "name": "Test Table 2", + "description": "", + "tags": [] } } ] @@ -236,7 +603,41 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00", + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema2.Test Table 2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema1.Test Table 1,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } @@ -253,7 +654,24 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "superset-2020_04_14-07_00_00", + "runId": "superset-2020_04_14-07_00_00-83v7ts", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_pipeline" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(superset,13)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": true + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "superset-2020_04_14-07_00_00-83v7ts", "lastRunId": "no-run-id-provided", "pipelineName": "test_pipeline" } diff --git a/metadata-ingestion/tests/integration/superset/test_superset.py b/metadata-ingestion/tests/integration/superset/test_superset.py index b3b59820161467..e8251e54a1f85a 100644 --- a/metadata-ingestion/tests/integration/superset/test_superset.py +++ b/metadata-ingestion/tests/integration/superset/test_superset.py @@ -133,6 +133,222 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: ], }, }, + "mock://mock-domain.superset.com/api/v1/dataset/": { + "method": "GET", + "status_code": 200, + "json": { + "count": 215, + "description_columns": {}, + "ids": [1, 2, 3], + "result": [ + { + "changed_by": { + "first_name": "Test", + "id": 1, + "last_name": "User1", + "username": "test_username_1", + }, + "changed_by_name": "test_username_1", + "changed_on_delta_humanized": "10 months ago", + "changed_on_utc": "2024-01-05T21:10:15.650819+0000", + "database": {"database_name": "test_database1", "id": 1}, + "datasource_type": "table", + "default_endpoint": None, + "description": None, + "explore_url": "/explore/?datasource_type=table&datasource_id=1", + "extra": None, + "id": 1, + "kind": "virtual", + "owners": [ + { + "first_name": "Test", + "id": 1, + "last_name": "Owner1", + "username": "test_username_1", + } + ], + "schema": "test_schema1", + "sql": "SELECT * FROM test_table1", + "table_name": "Test Table 1", + }, + { + "changed_by": { + "first_name": "Test", + "id": 2, + "last_name": "User2", + "username": "test_username_2", + }, + "changed_by_name": "test_username_2", + "changed_on_delta_humanized": "9 months ago", + "changed_on_utc": "2024-02-10T15:30:20.123456+0000", + "database": {"database_name": "test_database2", "id": 2}, + "datasource_type": "table", + "default_endpoint": None, + "description": "Sample description for dataset 2", + "explore_url": "/explore/?datasource_type=table&datasource_id=2", + "extra": None, + "id": 2, + "kind": "physical", + "owners": [ + { + "first_name": "Test", + "id": 2, + "last_name": "Owner2", + "username": "test_username_2", + } + ], + "schema": "test_schema2", + "sql": "SELECT * FROM test_table2", + "table_name": "Test Table 2", + }, + ], + }, + }, + "mock://mock-domain.superset.com/api/v1/dataset/1": { + "method": "GET", + "status_code": 200, + "json": { + "id": 1, + "result": { + "always_filter_main_dttm": False, + "cache_timeout": None, + "changed_by": {"first_name": "Test", "last_name": "User1"}, + "changed_on": "2024-01-05T21:10:15.650819+0000", + "changed_on_humanized": "10 months ago", + "created_by": {"first_name": "Test", "last_name": "User1"}, + "created_on": "2024-01-05T21:10:15.650819+0000", + "created_on_humanized": "10 months ago", + "currency_formats": {}, + "database": { + "backend": "postgresql", + "database_name": "test_database1", + "id": 1, + }, + "datasource_name": "Test Table 1", + "datasource_type": "table", + "default_endpoint": None, + "description": None, + "extra": None, + "fetch_values_predicate": None, + "filter_select_enabled": True, + "granularity_sqla": [ + ["created_at", "created_at"], + ["updated_at", "updated_at"], + ], + "id": 1, + "is_managed_externally": False, + "is_sqllab_view": False, + "kind": "virtual", + "main_dttm_col": None, + "metrics": [ + { + "changed_on": "2024-01-05T21:10:15.650819+0000", + "created_on": "2024-01-05T21:10:15.650819+0000", + "currency": None, + "d3format": None, + "description": None, + "expression": "count(*)", + "extra": None, + "id": 1, + "metric_name": "count", + "metric_type": None, + "rendered_expression": "count(*)", + "verbose_name": None, + "warning_text": None, + } + ], + "name": "Test Table 1", + "normalize_columns": True, + "offset": 0, + "owners": [{"first_name": "Test", "id": 1, "last_name": "Owner1"}], + "rendered_sql": "SELECT * FROM test_table1", + "schema": "test_schema1", + "select_star": "SELECT * FROM test_schema1.test_table1 LIMIT 100", + "sql": "SELECT * FROM test_table1", + "table_name": "Test Table 1", + "uid": "1__table", + "url": "/tablemodelview/edit/1", + "verbose_map": { + "__timestamp": "Time", + "id": "ID", + "name": "Name", + "created_at": "Created At", + "updated_at": "Updated At", + }, + }, + }, + }, + "mock://mock-domain.superset.com/api/v1/dataset/2": { + "method": "GET", + "status_code": 200, + "json": { + "id": 2, + "result": { + "always_filter_main_dttm": False, + "cache_timeout": None, + "changed_by": {"first_name": "Test", "last_name": "User2"}, + "changed_on": "2024-02-10T15:30:20.123456+0000", + "changed_on_humanized": "9 months ago", + "created_by": {"first_name": "Test", "last_name": "User2"}, + "created_on": "2024-02-10T15:30:20.123456+0000", + "created_on_humanized": "9 months ago", + "currency_formats": {}, + "database": { + "backend": "postgresql", + "database_name": "test_database1", + "id": 1, + }, + "datasource_name": "Test Table 2", + "datasource_type": "table", + "default_endpoint": None, + "description": "Sample description for dataset 2", + "extra": None, + "fetch_values_predicate": None, + "filter_select_enabled": True, + "granularity_sqla": [["date_column", "date_column"]], + "id": 2, + "is_managed_externally": False, + "is_sqllab_view": True, + "kind": "virtual", + "main_dttm_col": "date_column", + "metrics": [ + { + "changed_on": "2024-02-10T15:30:20.123456+0000", + "created_on": "2024-02-10T15:30:20.123456+0000", + "currency": None, + "d3format": None, + "description": None, + "expression": "sum(value)", + "extra": None, + "id": 2, + "metric_name": "total_value", + "metric_type": None, + "rendered_expression": "sum(value)", + "verbose_name": "Total Value", + "warning_text": None, + } + ], + "name": "Test Table 2", + "normalize_columns": True, + "offset": 0, + "owners": [{"first_name": "Test", "id": 2, "last_name": "Owner2"}], + "rendered_sql": "SELECT * FROM test_table2", + "schema": "test_schema2", + "select_star": "SELECT * FROM test_schema2.test_table2 LIMIT 100", + "sql": "SELECT * FROM test_table2", + "table_name": "Test Table 2", + "uid": "2__table", + "url": "/tablemodelview/edit/2", + "verbose_map": { + "__timestamp": "Time", + "id": "ID", + "name": "Name", + "value": "Value", + "date_column": "Date", + }, + }, + }, + }, "mock://mock-domain.superset.com/api/v1/dataset/20": { "method": "GET", "status_code": 200, @@ -147,6 +363,19 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: }, }, }, + "mock://mock-domain.superset.com/api/v1/database/1": { + "method": "GET", + "status_code": 200, + "json": { + "id": 1, + "result": { + "configuration_method": "sqlalchemy_form", + "database_name": "test_database1", + "id": 1, + "sqlalchemy_uri": "postgresql://user:password@host:port/test_database1", + }, + }, + }, "mock://mock-domain.superset.com/api/v1/database/30": { "method": "GET", "status_code": 200, @@ -225,6 +454,8 @@ def test_superset_stateful_ingest( "username": "test_username", "password": "test_password", "provider": "db", + # Enable dataset ingestion + "ingest_datasets": True, # enable stateful ingestion "stateful_ingestion": { "enabled": True, @@ -244,7 +475,7 @@ def test_superset_stateful_ingest( "pipeline_name": "test_pipeline", } - dashboard_endpoint_override = { + asset_override = { "mock://mock-domain.superset.com/api/v1/dashboard/": { "method": "GET", "status_code": 200, @@ -276,6 +507,92 @@ def test_superset_stateful_ingest( ], }, }, + "mock://mock-domain.superset.com/api/v1/chart/": { + "method": "GET", + "status_code": 200, + "json": { + "count": 3, + "result": [ + { + "id": "10", + "changed_by": { + "username": "test_username_1", + }, + "changed_on_utc": "2020-04-14T07:00:00.000000+0000", + "slice_name": "test_chart_title_1", + "viz_type": "box_plot", + "url": "/explore/test_chart_url_10", + "datasource_id": "20", + "params": '{"metrics": [], "adhoc_filters": []}', + }, + { + "id": "11", + "changed_by": { + "username": "test_username_1", + }, + "changed_on_utc": "2020-04-14T07:00:00.000000+0000", + "slice_name": "test_chart_title_2", + "viz_type": "pie", + "url": "/explore/test_chart_url_11", + "datasource_id": "20", + "params": '{"metrics": [], "adhoc_filters": []}', + }, + { + "id": "12", + "changed_by": { + "username": "test_username_2", + }, + "changed_on_utc": "2020-04-14T07:00:00.000000+0000", + "slice_name": "test_chart_title_3", + "viz_type": "treemap", + "url": "/explore/test_chart_url_12", + "datasource_id": "20", + "params": '{"metrics": [], "adhoc_filters": []}', + }, + ], + }, + }, + "mock://mock-domain.superset.com/api/v1/dataset/": { + "method": "GET", + "status_code": 200, + "json": { + "count": 214, + "description_columns": {}, + "ids": [1, 2], + "result": [ + { + "changed_by": { + "first_name": "Test", + "id": 2, + "last_name": "User2", + "username": "test_username_2", + }, + "changed_by_name": "test_username_2", + "changed_on_delta_humanized": "9 months ago", + "changed_on_utc": "2024-02-10T15:30:20.123456+0000", + "database": {"database_name": "test_database1", "id": 1}, + "datasource_type": "table", + "default_endpoint": None, + "description": "Sample description for dataset 2", + "explore_url": "/explore/?datasource_type=table&datasource_id=2", + "extra": None, + "id": 2, + "kind": "physical", + "owners": [ + { + "first_name": "Test", + "id": 2, + "last_name": "Owner2", + "username": "test_username_2", + } + ], + "schema": "test_schema2", + "sql": "SELECT * FROM test_table2", + "table_name": "Test Table 2", + }, + ], + }, + }, } with patch( @@ -292,10 +609,8 @@ def test_superset_stateful_ingest( assert checkpoint1 assert checkpoint1.state - # Remove one dashboard from the superset config. - register_mock_api( - request_mock=requests_mock, override_data=dashboard_endpoint_override - ) + # Remove one dashboard, chart, dataset from the superset config. + register_mock_api(request_mock=requests_mock, override_data=asset_override) # Capture MCEs of second run to validate Status(removed=true) deleted_mces_path = f"{tmp_path}/superset_deleted_mces.json" @@ -313,15 +628,27 @@ def test_superset_stateful_ingest( # part of the second state state1 = checkpoint1.state state2 = checkpoint2.state - difference_urns = list( + dashboard_difference_urns = list( state1.get_urns_not_in(type="dashboard", other_checkpoint_state=state2) ) + chart_difference_urns = list( + state1.get_urns_not_in(type="chart", other_checkpoint_state=state2) + ) + dataset_difference_urns = list( + state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) + ) - assert len(difference_urns) == 1 + assert len(dashboard_difference_urns) == 1 + assert len(chart_difference_urns) == 1 + assert len(dataset_difference_urns) == 1 urn1 = "urn:li:dashboard:(superset,2)" + urn2 = "urn:li:chart:(superset,13)" + urn3 = "urn:li:dataset:(urn:li:dataPlatform:postgres,test_database1.test_schema1.Test Table 1,PROD)" - assert urn1 in difference_urns + assert urn1 in dashboard_difference_urns + assert urn2 in chart_difference_urns + assert urn3 in dataset_difference_urns # Validate that all providers have committed successfully. validate_all_providers_have_committed_successfully( diff --git a/metadata-ingestion/tests/unit/test_preset_source.py b/metadata-ingestion/tests/unit/test_preset_source.py index d97db651f4c795..dc81f4c8284d50 100644 --- a/metadata-ingestion/tests/unit/test_preset_source.py +++ b/metadata-ingestion/tests/unit/test_preset_source.py @@ -20,3 +20,23 @@ def test_set_display_uri(): assert config.connect_uri == "" assert config.manager_uri == "https://api.app.preset.io" assert config.display_uri == display_uri + + +def test_preset_config_parsing(): + preset_config = { + "connect_uri": "https://preset.io", + "api_key": "dummy_api_key", + "api_secret": "dummy_api_secret", + "manager_uri": "https://api.app.preset.io", + } + + # Tests if SupersetConfig fields are parsed extra fields correctly + config = PresetConfig.parse_obj(preset_config) + + # Test Preset-specific fields + assert config.api_key == "dummy_api_key" + assert config.api_secret == "dummy_api_secret" + assert config.manager_uri == "https://api.app.preset.io" + + # Test that regular Superset fields are still parsed + assert config.connect_uri == "https://preset.io" From 1d5ddf0c041784a7a78630c232dd7e25aac6fa26 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Sat, 7 Dec 2024 13:40:32 +0100 Subject: [PATCH 149/174] fix(ingest/sagemaker): Adding option to control retry for any aws source (#8727) --- .../datahub/ingestion/source/aws/aws_common.py | 14 +++++++++++++- .../src/datahub/ingestion/source/aws/sagemaker.py | 8 ++++++++ .../source/aws/sagemaker_processors/common.py | 6 ++++++ .../source/aws/sagemaker_processors/jobs.py | 13 ++++++++++++- .../source/aws/sagemaker_processors/lineage.py | 15 +++++++++++---- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py index ce45a5c9b95dcc..161aed5bb59881 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union import boto3 from boto3.session import Session @@ -107,6 +107,14 @@ class AwsConnectionConfig(ConfigModel): default=None, description="A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", ) + aws_retry_num: int = Field( + default=5, + description="Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + ) + aws_retry_mode: Literal["legacy", "standard", "adaptive"] = Field( + default="standard", + description="Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + ) read_timeout: float = Field( default=DEFAULT_TIMEOUT, @@ -199,6 +207,10 @@ def _aws_config(self) -> Config: return Config( proxies=self.aws_proxy, read_timeout=self.read_timeout, + retries={ + "max_attempts": self.aws_retry_num, + "mode": self.aws_retry_mode, + }, **self.aws_advanced_config, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py index b63fa57f069b5b..55b8f4d889072d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker.py @@ -1,3 +1,4 @@ +import logging from collections import defaultdict from typing import TYPE_CHECKING, DefaultDict, Dict, Iterable, List, Optional @@ -36,6 +37,8 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient +logger = logging.getLogger(__name__) + @platform_name("SageMaker") @config_class(SagemakerSourceConfig) @@ -75,6 +78,7 @@ def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: ] def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: + logger.info("Starting SageMaker ingestion...") # get common lineage graph lineage_processor = LineageProcessor( sagemaker_client=self.sagemaker_client, env=self.env, report=self.report @@ -83,6 +87,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: # extract feature groups if specified if self.source_config.extract_feature_groups: + logger.info("Extracting feature groups...") feature_group_processor = FeatureGroupProcessor( sagemaker_client=self.sagemaker_client, env=self.env, report=self.report ) @@ -95,6 +100,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: # extract jobs if specified if self.source_config.extract_jobs is not False: + logger.info("Extracting jobs...") job_processor = JobProcessor( sagemaker_client=self.client_factory.get_client, env=self.env, @@ -109,6 +115,8 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: # extract models if specified if self.source_config.extract_models: + logger.info("Extracting models...") + model_processor = ModelProcessor( sagemaker_client=self.sagemaker_client, env=self.env, diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py index 45dadab7c24dff..73d8d33dd11be7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/common.py @@ -40,8 +40,11 @@ class SagemakerSourceReport(StaleEntityRemovalSourceReport): groups_scanned = 0 models_scanned = 0 jobs_scanned = 0 + jobs_processed = 0 datasets_scanned = 0 filtered: List[str] = field(default_factory=list) + model_endpoint_lineage = 0 + model_group_lineage = 0 def report_feature_group_scanned(self) -> None: self.feature_groups_scanned += 1 @@ -58,6 +61,9 @@ def report_group_scanned(self) -> None: def report_model_scanned(self) -> None: self.models_scanned += 1 + def report_job_processed(self) -> None: + self.jobs_processed += 1 + def report_job_scanned(self) -> None: self.jobs_scanned += 1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py index 73a83295ec8cba..be0a99c6d32346 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/jobs.py @@ -1,3 +1,4 @@ +import logging from collections import defaultdict from dataclasses import dataclass, field from enum import Enum @@ -49,6 +50,8 @@ if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient +logger = logging.getLogger(__name__) + JobInfo = TypeVar( "JobInfo", AutoMlJobInfo, @@ -274,15 +277,18 @@ def get_job_details(self, job_name: str, job_type: JobType) -> Dict[str, Any]: ) def get_workunits(self) -> Iterable[MetadataWorkUnit]: + logger.info("Getting all SageMaker jobs") jobs = self.get_all_jobs() processed_jobs: Dict[str, SageMakerJob] = {} + logger.info("Processing SageMaker jobs") # first pass: process jobs and collect datasets used + logger.info("first pass: process jobs and collect datasets used") for job in jobs: job_type = job_type_to_info[job["type"]] job_name = job[job_type.list_name_key] - + logger.debug(f"Processing job {job_name} with type {job_type}") job_details = self.get_job_details(job_name, job["type"]) processed_job = getattr(self, job_type.processor)(job_details) @@ -293,6 +299,9 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: # second pass: # - move output jobs to inputs # - aggregate i/o datasets + logger.info( + "second pass: move output jobs to inputs and aggregate i/o datasets" + ) for job_urn in sorted(processed_jobs): processed_job = processed_jobs[job_urn] @@ -301,6 +310,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: all_datasets.update(processed_job.input_datasets) all_datasets.update(processed_job.output_datasets) + self.report.report_job_processed() # yield datasets for dataset_urn, dataset in all_datasets.items(): @@ -322,6 +332,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.report.report_dataset_scanned() # third pass: construct and yield MCEs + logger.info("third pass: construct and yield MCEs") for job_urn in sorted(processed_jobs): processed_job = processed_jobs[job_urn] job_snapshot = processed_job.job_snapshot diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py index b677dccad24ac4..24e5497269c738 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/sagemaker_processors/lineage.py @@ -1,3 +1,4 @@ +import logging from collections import defaultdict from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, DefaultDict, Dict, List, Set @@ -6,6 +7,8 @@ SagemakerSourceReport, ) +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from mypy_boto3_sagemaker import SageMakerClient from mypy_boto3_sagemaker.type_defs import ( @@ -88,7 +91,6 @@ def get_all_contexts(self) -> List["ContextSummaryTypeDef"]: paginator = self.sagemaker_client.get_paginator("list_contexts") for page in paginator.paginate(): contexts += page["ContextSummaries"] - return contexts def get_incoming_edges(self, node_arn: str) -> List["AssociationSummaryTypeDef"]: @@ -225,27 +227,32 @@ def get_lineage(self) -> LineageInfo: """ Get the lineage of all artifacts in SageMaker. """ - + logger.info("Getting lineage for SageMaker artifacts...") + logger.info("Getting all actions") for action in self.get_all_actions(): self.nodes[action["ActionArn"]] = {**action, "node_type": "action"} + logger.info("Getting all artifacts") for artifact in self.get_all_artifacts(): self.nodes[artifact["ArtifactArn"]] = {**artifact, "node_type": "artifact"} + logger.info("Getting all contexts") for context in self.get_all_contexts(): self.nodes[context["ContextArn"]] = {**context, "node_type": "context"} + logger.info("Getting lineage for model deployments and model groups") for node_arn, node in self.nodes.items(): + logger.debug(f"Getting lineage for node {node_arn}") # get model-endpoint lineage if ( node["node_type"] == "action" and node.get("ActionType") == "ModelDeployment" ): self.get_model_deployment_lineage(node_arn) - + self.report.model_endpoint_lineage += 1 # get model-group lineage if ( node["node_type"] == "context" and node.get("ContextType") == "ModelGroup" ): self.get_model_group_lineage(node_arn, node) - + self.report.model_group_lineage += 1 return self.lineage_info From 70bec480888e71ed4bbfbc62cc56fab4cc46abe8 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Sat, 7 Dec 2024 13:45:14 +0100 Subject: [PATCH 150/174] fix(ingest/gc): Additional dataprocess cleanup fixes (#12049) --- .../source/gc/dataprocess_cleanup.py | 31 +++++++----- metadata-ingestion/tests/unit/test_gc.py | 48 +++++++++++++++++++ 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 3e51b7da9e8be1..8aacf13cdb00fb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -207,6 +207,9 @@ def fetch_dpis(self, job_urn: str, batch_size: int) -> List[dict]: assert self.ctx.graph dpis = [] start = 0 + # This graphql endpoint doesn't support scrolling and therefore after 10k DPIs it causes performance issues on ES + # Therefore, we are limiting the max DPIs to 9000 + max_item = 9000 while True: try: job_query_result = self.ctx.graph.execute_graphql( @@ -226,10 +229,12 @@ def fetch_dpis(self, job_urn: str, batch_size: int) -> List[dict]: runs = runs_data.get("runs") dpis.extend(runs) start += batch_size - if len(runs) < batch_size: + if len(runs) < batch_size or start >= max_item: break except Exception as e: - logger.error(f"Exception while fetching DPIs for job {job_urn}: {e}") + self.report.failure( + f"Exception while fetching DPIs for job {job_urn}:", exc=e + ) break return dpis @@ -254,8 +259,9 @@ def keep_last_n_dpi( deleted_count_last_n += 1 futures[future]["deleted"] = True except Exception as e: - logger.error(f"Exception while deleting DPI: {e}") - + self.report.report_failure( + f"Exception while deleting DPI: {e}", exc=e + ) if deleted_count_last_n % self.config.batch_size == 0: logger.info(f"Deleted {deleted_count_last_n} DPIs from {job.urn}") if self.config.delay: @@ -289,7 +295,7 @@ def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None: dpis = self.fetch_dpis(job.urn, self.config.batch_size) dpis.sort( key=lambda x: x["created"]["time"] - if "created" in x and "time" in x["created"] + if x.get("created") and x["created"].get("time") else 0, reverse=True, ) @@ -325,8 +331,8 @@ def remove_old_dpis( continue if ( - "created" not in dpi - or "time" not in dpi["created"] + not dpi.get("created") + or not dpi["created"].get("time") or dpi["created"]["time"] < retention_time * 1000 ): future = executor.submit( @@ -340,7 +346,7 @@ def remove_old_dpis( deleted_count_retention += 1 futures[future]["deleted"] = True except Exception as e: - logger.error(f"Exception while deleting DPI: {e}") + self.report.report_failure(f"Exception while deleting DPI: {e}", exc=e) if deleted_count_retention % self.config.batch_size == 0: logger.info( @@ -351,9 +357,12 @@ def remove_old_dpis( logger.info(f"Sleeping for {self.config.delay} seconds") time.sleep(self.config.delay) - logger.info( - f"Deleted {deleted_count_retention} DPIs from {job.urn} due to retention" - ) + if deleted_count_retention > 0: + logger.info( + f"Deleted {deleted_count_retention} DPIs from {job.urn} due to retention" + ) + else: + logger.debug(f"No DPIs to delete from {job.urn} due to retention") def get_data_flows(self) -> Iterable[DataFlowEntity]: assert self.ctx.graph diff --git a/metadata-ingestion/tests/unit/test_gc.py b/metadata-ingestion/tests/unit/test_gc.py index 5429c85dd608dc..8f00d5e064db85 100644 --- a/metadata-ingestion/tests/unit/test_gc.py +++ b/metadata-ingestion/tests/unit/test_gc.py @@ -84,6 +84,54 @@ def test_delete_dpi_from_datajobs_without_dpi_created_time(self, mock_fetch_dpis self.cleanup.delete_dpi_from_datajobs(job) self.assertEqual(10, self.report.num_aspects_removed) + @patch( + "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis" + ) + def test_delete_dpi_from_datajobs_without_dpi_null_created_time( + self, mock_fetch_dpis + ): + job = DataJobEntity( + urn="urn:li:dataJob:1", + flow_urn="urn:li:dataFlow:1", + lastIngested=int(datetime.now(timezone.utc).timestamp()), + jobId="job1", + dataPlatformInstance="urn:li:dataPlatformInstance:1", + total_runs=10, + ) + mock_fetch_dpis.return_value = [ + {"urn": f"urn:li:dataprocessInstance:{i}"} for i in range(10) + ] + [ + { + "urn": "urn:li:dataprocessInstance:11", + "created": {"time": None}, + } + ] + self.cleanup.delete_dpi_from_datajobs(job) + self.assertEqual(11, self.report.num_aspects_removed) + + @patch( + "datahub.ingestion.source.gc.dataprocess_cleanup.DataProcessCleanup.fetch_dpis" + ) + def test_delete_dpi_from_datajobs_without_dpi_without_time(self, mock_fetch_dpis): + job = DataJobEntity( + urn="urn:li:dataJob:1", + flow_urn="urn:li:dataFlow:1", + lastIngested=int(datetime.now(timezone.utc).timestamp()), + jobId="job1", + dataPlatformInstance="urn:li:dataPlatformInstance:1", + total_runs=10, + ) + mock_fetch_dpis.return_value = [ + {"urn": f"urn:li:dataprocessInstance:{i}"} for i in range(10) + ] + [ + { + "urn": "urn:li:dataprocessInstance:11", + "created": None, + } + ] + self.cleanup.delete_dpi_from_datajobs(job) + self.assertEqual(11, self.report.num_aspects_removed) + def test_fetch_dpis(self): assert self.cleanup.ctx.graph self.cleanup.ctx.graph = MagicMock() From 0e7ebaf0b2d08a2b21ed4351b60585a1f98065af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Mon, 9 Dec 2024 08:29:42 +0100 Subject: [PATCH 151/174] feat(tableau): adds more reporting metrics to better understand lineage construction in tableau ingestion (#12008) Co-authored-by: Harshal Sheth --- .../ingestion/source/tableau/tableau.py | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 197e73dca7141b..1dc32548e2eec4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -596,7 +596,15 @@ class TableauSourceReport(StaleEntityRemovalSourceReport): num_datasource_field_skipped_no_name: int = 0 num_csql_field_skipped_no_name: int = 0 num_table_field_skipped_no_name: int = 0 + # lineage + num_tables_with_upstream_lineage: int = 0 + num_upstream_table_lineage: int = 0 + num_upstream_fine_grained_lineage: int = 0 num_upstream_table_skipped_no_name: int = 0 + num_upstream_table_skipped_no_columns: int = 0 + num_upstream_table_failed_generate_reference: int = 0 + num_upstream_table_lineage_failed_parse_sql: int = 0 + num_upstream_fine_grained_lineage_failed_parse_sql: int = 0 @platform_name("Tableau") @@ -1311,7 +1319,7 @@ def _create_upstream_table_lineage( datasource: dict, browse_path: Optional[str], is_embedded_ds: bool = False, - ) -> Tuple: + ) -> Tuple[List[Upstream], List[FineGrainedLineage]]: upstream_tables: List[Upstream] = [] fine_grained_lineages: List[FineGrainedLineage] = [] table_id_to_urn = {} @@ -1472,6 +1480,7 @@ def get_upstream_tables( c.COLUMNS_CONNECTION ].get("totalCount") if not is_custom_sql and not num_tbl_cols: + self.report.num_upstream_table_skipped_no_columns += 1 logger.warning( f"Skipping upstream table with id {table[c.ID]}, no columns: {table}" ) @@ -1488,6 +1497,7 @@ def get_upstream_tables( table, default_schema_map=self.config.default_schema_map ) except Exception as e: + self.report.num_upstream_table_failed_generate_reference += 1 self.report.warning( title="Potentially Missing Lineage Issue", message="Failed to generate upstream reference", @@ -1659,15 +1669,7 @@ def get_upstream_fields_from_custom_sql( func_overridden_info=None, # Here we don't want to override any information from configuration ) - if parsed_result is None: - logger.info( - f"Failed to extract column level lineage from datasource {datasource_urn}" - ) - return [] - if parsed_result.debug_info.error: - logger.info( - f"Failed to extract column level lineage from datasource {datasource_urn}: {parsed_result.debug_info.error}" - ) + if parsed_result is None or parsed_result.debug_info.error: return [] cll: List[ColumnLineageInfo] = ( @@ -2031,6 +2033,8 @@ def _create_lineage_to_upstream_tables( aspect_name=c.UPSTREAM_LINEAGE, aspect=upstream_lineage, ) + self.report.num_tables_with_upstream_lineage += 1 + self.report.num_upstream_table_lineage += len(upstream_tables) @staticmethod def _clean_tableau_query_parameters(query: str) -> str: @@ -2130,7 +2134,7 @@ def parse_custom_sql( f"Overridden info upstream_db={upstream_db}, platform_instance={platform_instance}, platform={platform}" ) - return create_lineage_sql_parsed_result( + parsed_result = create_lineage_sql_parsed_result( query=query, default_db=upstream_db, platform=platform, @@ -2140,6 +2144,21 @@ def parse_custom_sql( schema_aware=not self.config.sql_parsing_disable_schema_awareness, ) + assert parsed_result is not None + + if parsed_result.debug_info.table_error: + logger.warning( + f"Failed to extract table lineage from datasource {datasource_urn}: {parsed_result.debug_info.table_error}" + ) + self.report.num_upstream_table_lineage_failed_parse_sql += 1 + elif parsed_result.debug_info.column_error: + logger.warning( + f"Failed to extract column level lineage from datasource {datasource_urn}: {parsed_result.debug_info.column_error}" + ) + self.report.num_upstream_fine_grained_lineage_failed_parse_sql += 1 + + return parsed_result + def _enrich_database_tables_with_parsed_schemas( self, parsing_result: SqlParsingResult ) -> None: @@ -2174,9 +2193,6 @@ def _create_lineage_from_unsupported_csql( ) if parsed_result is None: - logger.info( - f"Failed to extract table level lineage for datasource {csql_urn}" - ) return self._enrich_database_tables_with_parsed_schemas(parsed_result) @@ -2196,12 +2212,14 @@ def _create_lineage_from_unsupported_csql( upstreams=upstream_tables, fineGrainedLineages=fine_grained_lineages, ) - yield self.get_metadata_change_proposal( csql_urn, aspect_name=c.UPSTREAM_LINEAGE, aspect=upstream_lineage, ) + self.report.num_tables_with_upstream_lineage += 1 + self.report.num_upstream_table_lineage += len(upstream_tables) + self.report.num_upstream_fine_grained_lineage += len(fine_grained_lineages) def _get_schema_metadata_for_datasource( self, datasource_fields: List[dict] @@ -2352,6 +2370,11 @@ def emit_datasource( aspect_name=c.UPSTREAM_LINEAGE, aspect=upstream_lineage, ) + self.report.num_tables_with_upstream_lineage += 1 + self.report.num_upstream_table_lineage += len(upstream_tables) + self.report.num_upstream_fine_grained_lineage += len( + fine_grained_lineages + ) # Datasource Fields schema_metadata = self._get_schema_metadata_for_datasource( From 4811de1c1dc32995b4cc72da869c88483fdfbcb5 Mon Sep 17 00:00:00 2001 From: haeniya Date: Mon, 9 Dec 2024 09:59:11 +0100 Subject: [PATCH 152/174] feat(ingestion/tableau): hidden asset handling (#11559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yanik Häni --- .../ingestion/source/tableau/tableau.py | 84 +- .../tableau/tableau_cll_mces_golden.json | 197 +- ...bleau_extract_all_project_mces_golden.json | 197 +- ...tableau_hidden_asset_tags_mces_golden.json | 44892 ++++++++++++++++ ...leau_ingest_tags_disabled_mces_golden.json | 32956 ++++++++++++ .../tableau/tableau_mces_golden.json | 197 +- .../tableau_multiple_sites_mces_golden.json | 386 +- .../tableau_nested_project_mces_golden.json | 197 +- .../tableau_no_hidden_assets_mces_golden.json | 34728 ++++++++++++ ...leau_permission_ingestion_mces_golden.json | 193 +- .../tableau_signout_timeout_mces_golden.json | 197 +- ...tableau_site_name_pattern_mces_golden.json | 193 +- ...ableau_sites_as_container_mces_golden.json | 193 +- ...au_with_platform_instance_mces_golden.json | 193 +- .../tableau/test_tableau_ingest.py | 113 +- 15 files changed, 114839 insertions(+), 77 deletions(-) create mode 100644 metadata-ingestion/tests/integration/tableau/tableau_hidden_asset_tags_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/tableau/tableau_ingest_tags_disabled_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/tableau/tableau_no_hidden_assets_mces_golden.json diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 1dc32548e2eec4..68c38d4d064612 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -485,6 +485,18 @@ class TableauConfig( description="Configuration settings for ingesting Tableau groups and their capabilities as custom properties.", ) + ingest_hidden_assets: bool = Field( + True, + description="When enabled, hidden views and dashboards are ingested into Datahub. " + "If a dashboard or view is hidden in Tableau the luid is blank. Default of this config field is True.", + ) + + tags_for_hidden_assets: List[str] = Field( + default=[], + description="Tags to be added to hidden dashboards and views. If a dashboard or view is hidden in Tableau the luid is blank. " + "This can only be used with ingest_tags enabled as it will overwrite tags entered from the UI.", + ) + # pre = True because we want to take some decision before pydantic initialize the configuration to default values @root_validator(pre=True) def projects_backward_compatibility(cls, values: Dict) -> Dict: @@ -510,6 +522,20 @@ def projects_backward_compatibility(cls, values: Dict) -> Dict: return values + @root_validator() + def validate_config_values(cls, values: Dict) -> Dict: + tags_for_hidden_assets = values.get("tags_for_hidden_assets") + ingest_tags = values.get("ingest_tags") + if ( + not ingest_tags + and tags_for_hidden_assets + and len(tags_for_hidden_assets) > 0 + ): + raise ValueError( + "tags_for_hidden_assets is only allowed with ingest_tags enabled. Be aware that this will overwrite tags entered from the UI." + ) + return values + class WorkbookKey(ContainerKey): workbook_id: str @@ -605,6 +631,7 @@ class TableauSourceReport(StaleEntityRemovalSourceReport): num_upstream_table_failed_generate_reference: int = 0 num_upstream_table_lineage_failed_parse_sql: int = 0 num_upstream_fine_grained_lineage_failed_parse_sql: int = 0 + num_hidden_assets_skipped: int = 0 @platform_name("Tableau") @@ -1051,6 +1078,11 @@ def get_data_platform_instance(self) -> DataPlatformInstanceClass: ), ) + def _is_hidden_view(self, dashboard_or_view: Dict) -> bool: + # LUID is blank if the view is hidden in the workbook. + # More info here: https://help.tableau.com/current/api/metadata_api/en-us/reference/view.doc.html + return not dashboard_or_view.get(c.LUID) + def get_connection_object_page( self, query: str, @@ -2296,12 +2328,11 @@ def emit_datasource( ) # Tags - if datasource_info: + if datasource_info and self.config.ingest_tags: tags = self.get_tags(datasource_info) - if tags: - dataset_snapshot.aspects.append( - builder.make_global_tag_aspect_with_tag_list(tags) - ) + dataset_snapshot.aspects.append( + builder.make_global_tag_aspect_with_tag_list(tags) + ) # Browse path if browse_path and is_embedded_ds and workbook and workbook.get(c.NAME): @@ -2692,7 +2723,13 @@ def emit_sheets(self) -> Iterable[MetadataWorkUnit]: c.SHEETS_CONNECTION, sheets_filter, ): - yield from self.emit_sheets_as_charts(sheet, sheet.get(c.WORKBOOK)) + if self.config.ingest_hidden_assets or not self._is_hidden_view(sheet): + yield from self.emit_sheets_as_charts(sheet, sheet.get(c.WORKBOOK)) + else: + self.report.num_hidden_assets_skipped += 1 + logger.debug( + f"Skip view {sheet.get(c.ID)} because it's hidden (luid is blank)." + ) def emit_sheets_as_charts( self, sheet: dict, workbook: Optional[Dict] @@ -2783,11 +2820,17 @@ def emit_sheets_as_charts( chart_snapshot.aspects.append(owner) # Tags - tags = self.get_tags(sheet) - if tags: + if self.config.ingest_tags: + tags = self.get_tags(sheet) + if len(self.config.tags_for_hidden_assets) > 0 and self._is_hidden_view( + sheet + ): + tags.extend(self.config.tags_for_hidden_assets) + chart_snapshot.aspects.append( builder.make_global_tag_aspect_with_tag_list(tags) ) + yield self.get_metadata_change_event(chart_snapshot) if sheet_external_url is not None and self.config.ingest_embed_url is True: yield self.new_work_unit( @@ -2869,7 +2912,7 @@ def emit_workbook_as_container(self, workbook: Dict) -> Iterable[MetadataWorkUni else None ) - tags = self.get_tags(workbook) + tags = self.get_tags(workbook) if self.config.ingest_tags else None parent_key = None project_luid: Optional[str] = self._get_workbook_project_luid(workbook) @@ -3000,17 +3043,23 @@ def emit_dashboards(self) -> Iterable[MetadataWorkUnit]: c.DASHBOARDS_CONNECTION, dashboards_filter, ): - yield from self.emit_dashboard(dashboard, dashboard.get(c.WORKBOOK)) + if self.config.ingest_hidden_assets or not self._is_hidden_view(dashboard): + yield from self.emit_dashboard(dashboard, dashboard.get(c.WORKBOOK)) + else: + self.report.num_hidden_assets_skipped += 1 + logger.debug( + f"Skip dashboard {dashboard.get(c.ID)} because it's hidden (luid is blank)." + ) - def get_tags(self, obj: dict) -> Optional[List[str]]: + def get_tags(self, obj: dict) -> List[str]: tag_list = obj.get(c.TAGS, []) - if tag_list and self.config.ingest_tags: + if tag_list: tag_list_str = [ t[c.NAME] for t in tag_list if t is not None and t.get(c.NAME) ] return tag_list_str - return None + return [] def emit_dashboard( self, dashboard: dict, workbook: Optional[Dict] @@ -3061,8 +3110,13 @@ def emit_dashboard( ) dashboard_snapshot.aspects.append(dashboard_info_class) - tags = self.get_tags(dashboard) - if tags: + if self.config.ingest_tags: + tags = self.get_tags(dashboard) + if len(self.config.tags_for_hidden_assets) > 0 and self._is_hidden_view( + dashboard + ): + tags.extend(self.config.tags_for_hidden_assets) + dashboard_snapshot.aspects.append( builder.make_global_tag_aspect_with_tag_list(tags) ) diff --git a/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json index 855f8728380529..5901f18bd66433 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_cll_mces_golden.json @@ -870,8 +870,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -939,6 +939,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1221,6 +1226,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1789,6 +1799,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2409,6 +2424,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2925,6 +2945,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3077,6 +3102,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3604,6 +3634,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4112,6 +4147,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4588,6 +4628,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5093,6 +5138,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5485,6 +5535,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5851,6 +5906,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6246,6 +6306,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6693,6 +6758,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7111,6 +7181,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7451,6 +7526,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7817,6 +7897,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8079,6 +8164,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8526,6 +8616,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8869,6 +8964,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9238,6 +9338,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9630,6 +9735,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10077,6 +10187,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10227,8 +10342,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10277,6 +10392,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10384,6 +10504,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10489,6 +10614,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10608,6 +10738,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11038,6 +11173,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -12896,7 +13036,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", "type": "TRANSFORMED" }, { @@ -12904,7 +13044,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" } ], @@ -12995,6 +13135,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13336,6 +13481,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14251,6 +14401,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15212,6 +15367,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22084,6 +22244,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26204,6 +26369,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31558,6 +31728,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31902,6 +32077,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32671,6 +32851,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json index 3fd5ed1ac9dee6..f5914d5b2e2d1e 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_extract_all_project_mces_golden.json @@ -1129,8 +1129,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -1198,6 +1198,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1480,6 +1485,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2048,6 +2058,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2668,6 +2683,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3184,6 +3204,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3336,6 +3361,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3863,6 +3893,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4371,6 +4406,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4847,6 +4887,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5352,6 +5397,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5744,6 +5794,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6110,6 +6165,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6505,6 +6565,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6952,6 +7017,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7370,6 +7440,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7710,6 +7785,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8076,6 +8156,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8338,6 +8423,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8785,6 +8875,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9128,6 +9223,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9497,6 +9597,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9889,6 +9994,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10336,6 +10446,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10486,8 +10601,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10536,6 +10651,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10643,6 +10763,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10748,6 +10873,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10867,6 +10997,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11297,6 +11432,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13155,7 +13295,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", "type": "TRANSFORMED" }, { @@ -13163,7 +13303,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" } ], @@ -13254,6 +13394,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13595,6 +13740,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14510,6 +14660,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15471,6 +15626,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22343,6 +22503,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26463,6 +26628,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31817,6 +31987,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32161,6 +32336,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32930,6 +33110,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_hidden_asset_tags_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_hidden_asset_tags_mces_golden.json new file mode 100644 index 00000000000000..78c72f9d2a8710 --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_hidden_asset_tags_mces_golden.json @@ -0,0 +1,44892 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "79d02655-88e5-45a6-9f9b-eeaf5fe54903" + }, + "name": "DenyProject" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "ee012e36-d916-4c21-94ab-f0d66736af4e" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Deny Pattern WorkBook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + }, + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:hidden" + }, + { + "tag": "urn:li:tag:private" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:tag on published datasource" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems", + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "ATTRIBUTE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "BINFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "CALCULATEDFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COLUMNFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COUNT" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DATASOURCEFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DIMENSION" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "GROUPFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "HIERARCHYFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MEASURE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SETFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SUM" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "TagSheet3" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:YEAR", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "YEAR" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:hidden", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "hidden" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:private", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "private" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:tag on published datasource", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "tag on published datasource" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_hidden_asset_tags_ingest" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_ingest_tags_disabled_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_ingest_tags_disabled_mces_golden.json new file mode 100644 index 00000000000000..b0946fd053f756 --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_ingest_tags_disabled_mces_golden.json @@ -0,0 +1,32956 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Campaign%20List", + "title": "Campaign List", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Summary", + "title": "Summary", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)", + "schemaField": { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Names)", + "schemaField": { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Measure Values)", + "schemaField": { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Mobile%20-%20Sent%20by%20Campaign", + "title": "Mobile - Sent by Campaign", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)", + "schemaField": { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "schemaField": { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "schemaField": { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)", + "schemaField": { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Click Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Email Delivered%29)", + "schemaField": { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id %28Activity - Open Email%29)", + "schemaField": { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)", + "schemaField": { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "schemaField": { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "schemaField": { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet1", + "title": "Sheet 1", + "description": "", + "lastModified": { + "created": { + "time": 1639772911000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet2", + "title": "Sheet 2", + "description": "", + "lastModified": { + "created": { + "time": 1639773415000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Custom SQL Query)", + "schemaField": { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)", + "schemaField": { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)", + "schemaField": { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)", + "schemaField": { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)", + "schemaField": { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)", + "schemaField": { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Sheet3", + "title": "Sheet 3", + "description": "", + "lastModified": { + "created": { + "time": 1640375456000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)", + "schemaField": { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)", + "schemaField": { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)", + "schemaField": { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Requests", + "title": "Opened Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)", + "schemaField": { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)", + "schemaField": { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Top%2010%20Items%20by%20Requests%20and%20YoY%20Change", + "title": "Top 10 Items by Requests and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Problems", + "title": "Opened Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)", + "schemaField": { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)", + "schemaField": { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue", + "title": "Overdue", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/High%20and%20Critical%20Priority%20Problems", + "title": "High and Critical Priority Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)", + "schemaField": { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Total%20Incidents%20by%20Category%20and%20YoY%20Change", + "title": "Total Incidents by Category and YoY Change", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Known%20Errors", + "title": "Known Errors", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)", + "schemaField": { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "schemaField": { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)", + "schemaField": { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "schemaField": { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Requests", + "title": "Overdue Requests", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/AVG%20Time%20to%20Solve%20an%20Incident", + "title": "AVG Time to Solve an Incident", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)", + "schemaField": { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)", + "schemaField": { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Made%20SLA%3F", + "title": "Made SLA?", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)", + "schemaField": { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "schemaField": { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-IncidentBreakdownbyPriority", + "title": "Tooltip - Incident Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Overdue%20Problems", + "title": "Overdue Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)", + "schemaField": { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-ProblemBreakdownbyPriority", + "title": "Tooltip - Problem Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/Tooltip-RequestBreakdownbyPriority", + "title": "Tooltip - Request Breakdown by Priority", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Age%20of%20Active%20Problems", + "title": "Age of Active Problems", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)" + }, + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)", + "schemaField": { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)", + "schemaField": { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/t/acryl/authoring/ExecutiveDashboard/ExecutiveDashboard/Opened%20Incidents", + "title": "Opened Incidents", + "description": "", + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Category %28Incident%29)", + "schemaField": { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "schemaField": { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)", + "schemaField": { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)", + "schemaField": { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "schemaField": { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "schemaField": { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)", + "schemaField": { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "schemaField": { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "schemaField": { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Priority)", + "schemaField": { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)", + "schemaField": { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": {}, + "externalUrl": "https://do-not-connect/#/site/acryl/views/Workbookpublishedds/Sheet1", + "title": "published sheet ds", + "description": "", + "lastModified": { + "created": { + "time": 1641951867000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642658093000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)", + "schemaField": { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)", + "schemaField": { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)", + "schemaField": { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "dvd Rental Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Story 1", + "description": "", + "charts": [], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 1639773866000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1642199995000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": {}, + "title": "Executive Dashboard", + "description": "", + "charts": [ + "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 1639768450000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1639768502000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems", + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,130496dc-29ca-8a89-e32b-d73c4d8b65ff)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,20fc5eb7-81eb-aa18-8c39-af501c62d085)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b5351c1-535d-4a4a-1339-c51ddd6abf8a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,2b73b9dd-4ec7-75ca-f2e9-fa1984ca8b72)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,373c6466-bb0c-b319-8752-632456349261)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,53b8dc2f-8ada-51f7-7422-fe82e9b803cc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,58af9ecf-b839-da50-65e1-2e1fa20e3362)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,618b3e76-75c1-cb31-0c61-3f4890b72c31)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,721c3c41-7a2b-16a8-3281-6f948a44be96)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7ef184c1-5a41-5ec8-723e-ae44c20aa335)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,7fbc77ba-0ab6-3727-0db3-d8402a804da5)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8385ea9a-0749-754f-7ad9-824433de2120)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b207c2f2-b675-32e3-2663-17bb836a018b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,b679da5e-7d03-f01e-b2ea-01fb3c1926dc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c14973c2-e1c3-563a-a9c1-8a408396d22a)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,c57a5574-db47-46df-677f-0b708dab14db)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e604255e-0573-3951-6db7-05bee48116c1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,e70a540d-55ed-b9cc-5a3c-01ebe81a1274)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,20e44c22-1ccd-301a-220c-7b6837d09a52)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,39b7a1de-6276-cfc7-9b59-1d22f3bbb06b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,5dcaaf46-e6fb-2548-e763-272a7ab2c9b1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_ingest_tags_disabled" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json index 29e1b77869aa51..8610f809ac52b0 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_mces_golden.json @@ -870,8 +870,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -939,6 +939,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1221,6 +1226,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1789,6 +1799,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2409,6 +2424,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2925,6 +2945,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3077,6 +3102,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3604,6 +3634,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4112,6 +4147,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4588,6 +4628,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5093,6 +5138,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5485,6 +5535,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5851,6 +5906,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6246,6 +6306,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6693,6 +6758,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7111,6 +7181,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7451,6 +7526,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7817,6 +7897,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8079,6 +8164,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8526,6 +8616,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8869,6 +8964,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9238,6 +9338,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9630,6 +9735,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10077,6 +10187,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10227,8 +10342,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10277,6 +10392,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10384,6 +10504,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10489,6 +10614,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10608,6 +10738,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11038,6 +11173,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -12896,7 +13036,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", "type": "TRANSFORMED" }, { @@ -12904,7 +13044,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" } ], @@ -12995,6 +13135,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13336,6 +13481,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14251,6 +14401,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15212,6 +15367,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22084,6 +22244,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26204,6 +26369,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31558,6 +31728,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31902,6 +32077,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32671,6 +32851,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json index 12c665235e6ac7..05ec0e76f92d6d 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_multiple_sites_mces_golden.json @@ -1043,8 +1043,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -1112,6 +1112,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1398,6 +1403,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1970,6 +1980,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2594,6 +2609,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3114,6 +3134,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3270,6 +3295,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3805,6 +3835,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4317,6 +4352,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4797,6 +4837,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5306,6 +5351,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5702,6 +5752,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6072,6 +6127,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6471,6 +6531,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6922,6 +6987,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7344,6 +7414,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7688,6 +7763,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8058,6 +8138,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8324,6 +8409,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8775,6 +8865,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9122,6 +9217,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9495,6 +9595,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9891,6 +9996,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10342,6 +10452,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10496,8 +10611,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10546,6 +10661,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10657,6 +10777,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10766,6 +10891,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10889,6 +11019,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11323,6 +11458,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13284,6 +13424,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13629,6 +13774,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14548,6 +14698,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15513,6 +15668,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22389,6 +22549,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26513,6 +26678,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31871,6 +32041,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32219,6 +32394,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32996,6 +33176,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -44181,8 +44366,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -44250,6 +44435,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -44506,6 +44696,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -45048,6 +45243,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -45642,6 +45842,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -46132,6 +46337,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -46258,6 +46468,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -46733,6 +46948,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -47215,6 +47435,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -47665,6 +47890,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -48144,6 +48374,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -48510,6 +48745,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -48850,6 +49090,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -49219,6 +49464,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -49640,6 +49890,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -50032,6 +50287,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -50346,6 +50606,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -50686,6 +50951,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -50922,6 +51192,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -51343,6 +51618,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -51660,6 +51940,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -52003,6 +52288,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -52369,6 +52659,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -52790,6 +53085,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -52914,8 +53214,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -52964,6 +53264,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -53045,6 +53350,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -53124,6 +53434,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -53217,6 +53532,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -53621,6 +53941,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -55552,6 +55877,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -55867,6 +56197,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -56756,6 +57091,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -57691,6 +58031,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -64537,6 +64882,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -68631,6 +68981,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -73959,6 +74314,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -74277,6 +74637,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -74998,6 +75363,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json index 2aee4abc4d049f..dd852671d760c2 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_nested_project_mces_golden.json @@ -1129,8 +1129,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -1198,6 +1198,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1480,6 +1485,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2048,6 +2058,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2668,6 +2683,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3184,6 +3204,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3336,6 +3361,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3863,6 +3893,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4371,6 +4406,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4847,6 +4887,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5352,6 +5397,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5744,6 +5794,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6110,6 +6165,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6505,6 +6565,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6952,6 +7017,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7370,6 +7440,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7710,6 +7785,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8076,6 +8156,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8338,6 +8423,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8785,6 +8875,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9128,6 +9223,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9497,6 +9597,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9889,6 +9994,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10336,6 +10446,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10486,8 +10601,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10536,6 +10651,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10643,6 +10763,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10748,6 +10873,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10867,6 +10997,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11297,6 +11432,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13155,7 +13295,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", "type": "TRANSFORMED" }, { @@ -13163,7 +13303,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" } ], @@ -13254,6 +13394,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13595,6 +13740,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14510,6 +14660,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15471,6 +15626,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22343,6 +22503,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26463,6 +26628,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31817,6 +31987,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32161,6 +32336,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32930,6 +33110,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_no_hidden_assets_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_no_hidden_assets_mces_golden.json new file mode 100644 index 00000000000000..db1d7b4ba25d5f --- /dev/null +++ b/metadata-ingestion/tests/integration/tableau/tableau_no_hidden_assets_mces_golden.json @@ -0,0 +1,34728 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "190a6a5c-63ed-4de1-8045-faeae5df5b01" + }, + "name": "default" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "c30aafe5-44f4-4f28-80d3-d181010a263c" + }, + "name": "Project 2" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:252a054d4dd93cd657735aa46dd71370", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "910733aa-2e95-4ac3-a2e8-71570751099d" + }, + "name": "Samples" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "project_id": "79d02655-88e5-45a6-9f9b-eeaf5fe54903" + }, + "name": "DenyProject" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Project" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "1f15d897-7f0c-7c59-037a-afa6a9b7c9a9" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15995", + "name": "Email Performance by Campaign", + "description": "Description for Email Performance by Campaign" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "661fabd0-bed6-8610-e066-0694a81a6cea" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15619", + "name": "Dvdrental Workbook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:TagSheet3" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "6ffa5a7f-d852-78f1-6c6d-20ac23610ebf" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/15605", + "name": "Executive Dashboard", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "bd040833-8f66-22c0-1b51-bd4ccf5eef7c" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Workbook published ds", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "tableau", + "workbook_id": "ee012e36-d916-4c21-94ab-f0d66736af4e" + }, + "externalUrl": "https://do-not-connect/#/site/acryl/workbooks/17904", + "name": "Deny Pattern WorkBook", + "description": "" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:595877512935338b94eac9e06cf20607", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce", + "urn": "urn:li:container:beaddce9d1e89ab503ae6408fb77d4ce" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "chartUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 5 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.ChartSnapshot": { + "urn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.chart.ChartInfo": { + "customProperties": { + "luid": "f0779f9d-6765-47a9-a8f6-c740cfd27783" + }, + "externalUrl": "https://do-not-connect/t/acryl/authoring/EmailPerformancebyCampaign/EmailPerformancebyCampaign/Timeline%20-%20Sent", + "title": "Timeline - Sent", + "description": "", + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "inputFields", + "aspect": { + "json": { + "fields": [ + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Active)", + "schemaField": { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date)", + "schemaField": { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),ID)", + "schemaField": { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)", + "schemaField": { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Name)", + "schemaField": { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program Name)", + "schemaField": { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "dashboardUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1638860400000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "viewsCount": 3 + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DashboardSnapshot": { + "urn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { + "customProperties": { + "luid": "fc9ea488-f810-4fa8-ac19-aa96018b5d66" + }, + "title": "Email Performance by Campaign", + "description": "", + "charts": [ + "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "urn:li:chart:(tableau,38130558-4194-2e2a-3046-c0d887829cb4)", + "urn:li:chart:(tableau,692a2da4-2a82-32c1-f713-63b8e4325d86)", + "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" + ], + "datasets": [], + "lastModified": { + "created": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + }, + "lastModified": { + "time": 1640200234000, + "actor": "urn:li:corpuser:jawadqu@gmail.com" + } + }, + "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Activity_Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity Date %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Bounceback)" + ], + "transformOperation": "CalculatedFieldformula: [Sent Email] - [Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Campaign_Run_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Campaign Run ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Choice_Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Choice Number %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click-to-Open)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Click Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Click Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Rate)" + ], + "transformOperation": "CalculatedFieldformula: [Clickthrough Emails]/[Delivered Email]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Email Delivered)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Email Delivered)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivery Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Delivered Email]/[Sent Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Has_Predictive)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Is Predictive)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD),Mailing_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Mailing ID %28Activity - Email Delivered%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Non Clickthrough Email)" + ], + "transformOperation": "CalculatedFieldformula: [Delivered Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Delivered Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Rate)" + ], + "transformOperation": "CalculatedFieldformula: ZN([Opened Email]/[Delivered Email])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Activity)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Open Email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id (Activity - Open Email)])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Email)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Clickthrough Emails)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Opened Non Clicked Emails)" + ], + "transformOperation": "CalculatedFieldformula: [Opened Email]-[Clickthrough Emails]", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),Platform)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Platform %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Sent Email)" + ], + "transformOperation": "CalculatedFieldformula: COUNTD([Id])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),updatedAt)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Updated At)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD),User_Agent)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),User Agent %28Activity - Click Email%29)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD),programName)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD),Workspace Name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "Marketo", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Delivery Rate", + "nullable": false, + "description": "formula: ZN([Delivered Email]/[Sent Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workspace Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Rate", + "nullable": false, + "description": "formula: [Clickthrough Emails]/[Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Open Rate", + "nullable": false, + "description": "formula: ZN([Opened Email]/[Delivered Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sent Email", + "nullable": false, + "description": "formula: COUNTD([Id])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivered Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Email Delivered)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created At", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Clickthrough Emails", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Click Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Is Mobile Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing ID (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Click-to-Open", + "nullable": false, + "description": "formula: ZN([Clickthrough Emails]\r\n/ \r\n[Opened Email])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Email", + "nullable": false, + "description": "formula: COUNTD([Id (Activity - Open Email)])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign Run ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Non Clicked Emails", + "nullable": false, + "description": "formula: [Opened Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Clickthrough Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Clickthrough Emails]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lead ID (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Program Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign ID (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity Date (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Test Variant (Activity - Click Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Non Opened Email", + "nullable": false, + "description": "formula: [Delivered Email]-[Opened Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Has Predictive (Activity - Email Delivered)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User Agent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Step ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Bounceback", + "nullable": false, + "description": "formula: [Sent Email] - [Delivered Email]", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Choice Number (Activity - Open Email)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a", + "urn": "urn:li:container:008e111aa1d250dd52e0fd5d4b307b1a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),First Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),Last Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD),rental_id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Customer Payment Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "actor+ (dvdrental)", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Address2", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "District", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "country, city", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Phone", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Update (Address)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Address Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Actor Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "actor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Last Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City Id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),City)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Country/Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Customer Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Discount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Discount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Location)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Manufacturer)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID %28Returns%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID %28Returns%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Order ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Order ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Orders)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Orders)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Customer ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),People)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Postal Code)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product Name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Product)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit %28bin%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit Ratio)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Quantity)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Quantity)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region %28People%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region %28People%29)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Region)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Regional Manager)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Regional Manager)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returned)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returned)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Returns)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Returns)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Row ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Row ID)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sales)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Segment)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Ship Mode)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Ship Mode)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),State)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Sub-Category)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD),Top Customers by Profit)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Regional Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Made SLA)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% made SLA)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total # Request)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD),Total Active Requests)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T19:35:39Z", + "extractLastUpdateTime": "2018-01-18T19:35:39Z" + }, + "name": "Requests", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Closed by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% made SLA", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of requests which made SLA\r\n\r\nCOUNTD(IF [Made SLA]= TRUE\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Meta", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Request", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the total number of problems ignoring opened date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered item link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible elsewhere", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Update name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Special instructions", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Execution Plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Bundles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No search", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "List Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer update", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Guide", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Package", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Estimated Delivery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price Frequency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Recurring Price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fulfillment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Backordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ignore price", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an accident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\nIF [Active]=FALSE \r\nAND\r\n[Closed]>[Due date]\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\n[Active]=TRUE \r\nAND NOW()>[Due date]\r\n\r\nTHEN \"Overdue\"\r\nELSE \"Non Overdue\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Replace on upgrade", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Context", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requested for", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Billable (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Mobile Picture Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Protection policy", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Omit price in cart", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalogs", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Entitlement script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published version", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Preview link", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Price (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Visible on Guides", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Requested Item) 1", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Stage", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct request made in the last year. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve Time", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Icon", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Display name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No proceed checkout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Use cart layout", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "No order now", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Application", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Template", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Catalog", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Image", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents which are overdue\r\n\r\n(IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workflow", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created from item design", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Hide price (mobile listings)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Availability", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Picture", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan script", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Requested Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Requests", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct active request. The \"exclude\" is used to avoid filters interfering with the final result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Request state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Request)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Catalog Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Priority)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of critical and high priority)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Known error)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),% of known error)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "transformOperation": "CalculatedFieldformula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Related Incidents)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Problems with Related Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Age of Problems)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Time Span Breakdown)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total # Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD),Total Active Problems)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:21:33Z", + "extractLastUpdateTime": "2018-01-18T20:21:33Z" + }, + "name": "Problems", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "SLA due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Default assignee", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Workaround", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It checks if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])> max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active]=TRUE) \r\nAND NOW()> MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time Span Breakdown", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It groups problems accordingly with their ages\r\n\r\nIF [Age of Problems]< 2592000\r\nTHEN \"Under 30 d\"\r\nELSEIF [Age of Problems] > 7776000\r\nTHEN \"+ than 90d\"\r\nELSE \" 30-90 d\"\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "u_", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change request", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of incidents that are overdue\r\n\r\nIFNULL((IF ATTR([Overdue]= \"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND),0)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Roles", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total # Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct problems ignoring date\r\n\r\n{ EXCLUDE [Opened]: COUNTD([Number])}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Exclude manager", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lucha", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problems with Related Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct problems which has a related incident\r\n\r\nCOUNT(IF ([Related Incidents]>0\r\nAND NOT ISNULL([Related Incidents]))=true\r\nTHEN [Number]\r\nEND)", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Related Incidents", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of known error", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of problems that are known errors\r\n\r\nCOUNTD(IF [Known error]=TRUE\r\nTHEN [Number] END)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem state", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if the year of opened is equal the max year of the dataset \r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Known error", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group email", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Age of Problems", + "nullable": false, + "description": "formula: //Calculates the age of active problems since opened until now\r\n\r\nDATEDIFF('second', [Opened], NOW())", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of critical and high priority", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows the percentage of critical or high priority problems\r\n\r\nCOUNTD(IF [Priority]=1\r\nOR [Priority]=2\r\nTHEN [Number]\r\nEND)\r\n/\r\nCOUNTD([Number])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Include members", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each disctinct problem. The \"exclude\" is used to avoid filters interfering with the result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Problem)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Problems", + "nullable": false, + "description": "formula: // This is a calculated field using a level of detail calculation (LOD)\r\n// It counts each distinct active problem. The \"exclude\" is used to avoid filters interfering with the result \r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Group)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),% of Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Current Year Total Cases)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened Month Tooltip)" + ], + "transformOperation": "CalculatedFieldformula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Due date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Closed)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident %28seconds%29)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Time to Close an Incident)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Opened)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Overdue)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Active)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Max Year?)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD),Total Active Incidents)" + ], + "transformOperation": "CalculatedFieldformula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-01-18T20:13:08Z", + "extractLastUpdateTime": "2018-01-18T20:13:08Z" + }, + "name": "Incidents", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Assignment group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Attributes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "User input (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close code (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Migrated Data", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Notify (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Requires verification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Maintenance schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Escalation", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Warranty expiration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "GL account", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional assignee list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "First discovered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "% of Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It show the percentage incidents which are overdue\r\n\r\n(IF ATTR([Overdue]=\"Overdue\")\r\nTHEN COUNTD([Number])\r\nEND)\r\n/\r\nATTR({ EXCLUDE [Overdue]:\r\nCOUNTD([Number])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Skip sync", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "DNS Domain", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caller (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Department", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Current Year Total Cases", + "nullable": false, + "description": "formula: // This is a calculated field using level of detail calculation\r\n// It counts each distinct incident. The exclude is used to guarantee that filters will not interfere in the result\r\n\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Max Year?]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Close notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Managed by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Names", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "PO number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Short description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Child Incidents (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "IP Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Asset tag", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked out", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fully qualified domain name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Installed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Purchased", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Lease contract", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Vendor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Overdue", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows if an incident is overdue\r\n\r\n//Check overdue cases among inactive incidents\r\n{ FIXED [Number]:\r\n\r\nIF MAX([Active]=FALSE) \r\nAND\r\nmax([Closed])>max([Due date])\r\n\r\nOR\r\n//Check overdue cases among active incidents\r\nMAX([Active] = TRUE) \r\nAND NOW() > MAX([Due date]) \r\n\r\nTHEN \"Overdue\"\r\nEND\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval history", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent Incident (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost currency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "SLA due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Impact (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Monitor", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Serial number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Model ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Parent (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident (seconds)", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It calculates the difference in seconds between opening and closing an incident\r\n\r\n// Check if closed date is valid\r\nIF [Closed]>[Opened]\r\nTHEN\r\n//Calculate difference\r\nDATEDIFF('second', [Opened], [Closed])\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Owned by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Activity due", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Invoice number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval set (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Duration", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Start date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ordered", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business duration (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order received", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discovery source", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Total Active Incidents", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It counts each distinct incident. The \"exclude\" is used to avoid filters interfering in the final result\r\n\r\n{EXCLUDE [Opened], [Overdue], [Max Year?]: \r\nCOUNTD(IF [Active]=TRUE\r\nTHEN [Number]\r\nEND)\r\n}", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + }, + { + "tag": "urn:li:tag:ATTRIBUTE" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Class", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Operational status", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Expected start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work notes list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolve time (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reopen count (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created by (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned to (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Most recent discovery", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon reject (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Knowledge (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Max Year?", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It shows TRUE/FALSE if opened date equals maximum year in the dataset\r\n\r\nDATEPART(\"year\", [Opened]) = DATEPART(\"year\", {MAX([Opened])})", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Watch list (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery task", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Fault count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Caused by Change (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updated (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "MAC Address", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Approval", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Priority (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Urgency", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Company (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Additional comments (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Business service (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Schedule", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Supported by", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Configuration item (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Support group", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation display (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Justification", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Change Request (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Updates", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Incident state (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Made SLA (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Opened Month Tooltip", + "nullable": false, + "description": "formula: // This is an IF statment using the opened field to allow for different formats\r\n// original used on columns for line charts are formatted in starting letter of month i.e. J for January\r\n// This version formats to full month name for the tooltip \r\n\r\n\r\n// The IF statement names the month based on the month number of the datefield\r\nIF DATEPART('month', [Opened]) = 1 THEN 'January'\r\nELSEIF DATEPART('month', [Opened]) = 2 THEN 'February'\r\nELSEIF DATEPART('month', [Opened]) = 3 THEN 'March'\r\nELSEIF DATEPART('month', [Opened]) = 4 THEN 'April'\r\nELSEIF DATEPART('month', [Opened]) = 5 THEN 'May'\r\nELSEIF DATEPART('month', [Opened]) = 6 THEN 'June'\r\nELSEIF DATEPART('month', [Opened]) = 7 THEN 'July'\r\nELSEIF DATEPART('month', [Opened]) = 8 THEN 'August'\r\nELSEIF DATEPART('month', [Opened]) = 9 THEN 'September'\r\nELSEIF DATEPART('month', [Opened]) = 10 THEN 'October'\r\nELSEIF DATEPART('month', [Opened]) = 11 THEN 'Novemeber'\r\nELSEIF DATEPART('month', [Opened]) = 12 THEN 'December'\r\nELSE NULL\r\nEND", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Problem (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Measure Values", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Group list", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Checked in", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Severity (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number of Records", + "nullable": false, + "description": "formula: 1", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time worked (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Cost center", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work end (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Due date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Created (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Delivery plan", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Subcategory (Configuration Item)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sys ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments and Work notes (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Can Print", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Active", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "BOOLEAN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Number (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Follow up (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Task type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Domain Path (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Closed", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Description (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Reassignment count", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Contact type (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assignment group (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Comments", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Work start (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Correlation ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Resolved (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Assigned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Time to Close an Incident", + "nullable": false, + "description": "formula: // This is a calculated field\r\n// It transforms time in seconds into DD:HH:MM:SS format\r\nSTR(INT(AVG([Time to Close an Incident (seconds)])/86400)) \r\n \r\n+ \" day(s) \" + \r\n \r\nIF (INT(AVG([Time to Close an Incident (seconds)])%86400/3600)) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%86400/3600))\r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)])%3600/60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)])%3600/60)) \r\n \r\n \r\n+ \":\" + \r\n \r\nIF INT(AVG([Time to Close an Incident (seconds)]) %3600 %60) \r\n< 10 THEN \"0\" ELSE \"\" END + STR(INT(AVG([Time to Close an Incident (seconds)]) %3600 %60))", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Upon approval (Incident)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d", + "urn": "urn:li:container:047691e9c16bec8fb08e1df0f5d71c4d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),Custom SQL Query)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),Custom SQL Query)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),amount)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_id)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),customer_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),payment_date)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_first_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD),staff_last_name)" + ], + "transformOperation": "IDENTITY", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Custom SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DATASOURCEFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,c7dd65fb-6e7e-4091-bbde-8c78b34a40f8,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD),Program ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Workbook published ds" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "hasExtracts": "True", + "extractLastRefreshTime": "2018-02-09T00:05:25Z", + "extractLastUpdateTime": "2018-02-09T00:05:25Z" + }, + "name": "New DataSource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Program ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Embedded Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9", + "urn": "urn:li:container:94e6e84b66f9ee8c70c22f06cfbad6a9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),customer_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),customer_last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD),staff_last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD),staff_last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:tag on published datasource" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "test publish datasource", + "description": "description for test publish datasource", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "payment_date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Published SQL Query", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),City)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Postal Code)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Country/Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Region)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),State)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Location)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)" + ], + "transformOperation": "GroupField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sub-Category)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Manufacturer)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product Name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Product)" + ], + "transformOperation": "HierarchyField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit %28bin%29)" + ], + "transformOperation": "BinField", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Sales)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit Ratio)" + ], + "transformOperation": "CalculatedFieldformula: SUM([Profit])/SUM([Sales])", + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD),Segment)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Segment)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Profit)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD),Top Customers by Profit)" + ], + "transformOperation": "SetField", + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jawadqu@gmail.com", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Superstore Datasource", + "description": "Description for Superstore dataset", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Top Customers by Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:SETFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returns", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit Ratio", + "nullable": false, + "description": "formula: SUM([Profit])/SUM([Sales])", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:CALCULATEDFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Returned", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Orders", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Profit (bin)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:BINFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID (Returns)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Person", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Product", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Location", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "UNKNOWN", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:HIERARCHYFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "People", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": {} + } + }, + "nativeDataType": "TABLE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:COUNT" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Manufacturer", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:GROUPFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Region (People)", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "REAL", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:MEASURE" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:SUM" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + }, + { + "tag": "urn:li:tag:YEAR" + } + ] + }, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "description": "", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STRING", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:DIMENSION" + }, + { + "tag": "urn:li:tag:COLUMNFIELD" + } + ] + }, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Published Data Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873", + "urn": "urn:li:container:d2dcd6bd1bb954d62f1cfc68332ee873" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/Customer Payment Query" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tcustomer.customer_id,\n\tfirst_name,\n\tlast_name,\n\tamount,\n\tpayment_date,\n\trental_id\nFROM\n\tcustomer\nINNER JOIN payment \n ON payment.customer_id = customer.customer_id\nwhere customer.customer_id = <[Parameters].[Parameter 1]>\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + }, + { + "id": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1", + "urn": "urn:li:container:fad3de4b86519c3edeb685215fe0bab1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "NUMERIC", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "SELECT\n\tc.customer_id,\n\tc.first_name customer_first_name,\n\tc.last_name customer_last_name,\n\ts.first_name staff_first_name,\n\ts.last_name staff_last_name,\n\tamount,\n\tpayment_date\nFROM\n\tcustomer c\nINNER JOIN payment p \n ON p.customer_id = c.customer_id\nINNER JOIN staff s \n ON p.staff_id = s.staff_id\nORDER BY payment_date", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b", + "urn": "urn:li:container:5ec314b9630974ec084f5dfd3849f87b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:tableau" + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "seller_city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/SubProject1/AbcJoinWorkbook" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": {}, + "name": "Custom SQL Query", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "select seller_city, price from demo-custom-323403.bigquery_demo.sellers sell LEFT JOIN (\nselect * from demo-custom-323403.bigquery_demo.order_items\n) items on items.seller_id=sell.seller_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View", + "Custom SQL" + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "SubProject1" + }, + { + "id": "AbcJoinWorkbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Link_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Platform", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Is_Mobile_Device", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "User_Agent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Test_Variant", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_Run_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Activity_Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Mailing_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Has_Predictive", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Campaign_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Step_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Lead_ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Choice_Number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Email Performance by Campaign/Marketo" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "programName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "programId", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workspaceName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "updatedAt", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Email Performance by Campaign" + }, + { + "id": "Marketo" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I2", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Dvdrental Workbook/actor+ (dvdrental)" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "DBTIMESTAMP", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "STR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "actor_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I4", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Dvdrental Workbook" + }, + { + "id": "actor+ (dvdrental)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Person", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Returned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/Samples/Superstore Datasource" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "Product ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Postal Code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "City", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "State", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Country/Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sales", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Segment", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Sub-Category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Profit", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Product Name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Customer ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Order ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Row ID", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "I8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Discount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "R8", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Ship Mode", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "Region", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WSTR", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Samples" + }, + { + "id": "Superstore Datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems", + "/prod/tableau/default/Executive Dashboard/Requests", + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_for", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "requested_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "special_instructions", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "configuration_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_catalog", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stage", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "estimated_delivery", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "context", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cat_item", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "backordered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "request", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Requests" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sc_catalogs", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "workflow", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_customer_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_standalone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_quantity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_scope", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "template", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_proceed_checkout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billable", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "meta", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ordered_item_link", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_version", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "image", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_policy", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "list_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order_now", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sc_ic_item_staging", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "entitlement_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "icon", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ignore_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_closed", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_package", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_replace_on_upgrade", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_FLOAT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "use_sc_layout", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "availability", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "custom_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_picture", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_cart", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mobile_hide_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_time", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "no_search", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_frequency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "recurring_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan_script", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_bundle", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_update_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "visible_guide", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "preview", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "omit_price", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Requests" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "u_u", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "exclude_manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lucha", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "u_lu2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "roles", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "default_assignee", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manager", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "include_members", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Problems" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "related_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_around", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "known_error", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Problems" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urgency", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "severity", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_service", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_set", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent_incident", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caller_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "group_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_display", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "resolved_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "parent", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "user_input", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "escalation", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval_history", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "impact", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "expected_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "additional_assignee_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "child_incidents", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cmdb_ci", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "incident_state", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "notify", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reassignment_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "contact_type", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "problem_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "opened_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_notes_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "time_worked", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "caused_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_reject", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_task", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "knowledge", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "closed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_end", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "reopen_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "work_start", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "made_sla", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "calendar_stc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rfc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_plan", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "close_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activity_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "business_duration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "follow_up", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments_and_work_notes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "upon_approval", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "watch_list", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sla_due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Executive Dashboard/Incidents" + ] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test", + "platform": "urn:li:dataPlatform:tableau", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "operational_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_discovered", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_cc", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "attributes", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "serial_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "vendor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "ip_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "support_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "supported_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "invoice_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "managed_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fault_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due_in", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "correlation_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "justification", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_class_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "comments", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "lease_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "monitor", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cost_center", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "subcategory", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "delivery_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assignment_group", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "can_print", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "short_description", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "model_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "start_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "discovery_source", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_domain_path", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "assigned_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "category", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "fqdn", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "warranty_expiration", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "owned_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "asset_tag", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "manufacturer", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "purchase_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "WDC_DATE", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "location", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "department", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_updated_on", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "checked_out", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "unverified", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "skip_sync", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "WDC_BOOL", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "po_number", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "order_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "gl_account", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "maintenance_schedule", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dns_domain", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_created_by", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "mac_address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "change_control", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "WDC_STRING", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "install_status", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "due", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "WDC_DATETIME", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "sys_mod_count", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "WDC_INT", + "recursive": false, + "isPartOfKey": false + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Executive Dashboard" + }, + { + "id": "Incidents" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/Customer Payment Query", + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "Customer Payment Query" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/tableau/default/test publish datasource" + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "default" + }, + { + "id": "test publish datasource" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(tableau,222d1406-de0e-cd8d-0b94-9b45a0007e59)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(tableau,8f7dd564-36b6-593f-3c6f-687ad06cd40b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.people,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:external,sample - superstore%2C %28new%29.xls.returns,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.payment,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,demo_postgres_instance.dvdrental.public.staff,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,00cce29f-b561-bb41-3557-8e19660bb5dd,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,06c3e060-8133-4b58-9b53-a0fced25e056,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,3ade7817-ae27-259e-8e48-1570e7f932f6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4644ccb1-2adc-cf26-c654-04ed1dcc7090,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,4fb670d5-3e19-9656-e684-74aa9729cf18,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,5449c627-7462-4ef7-b492-bda46be068e3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,618c87db-5959-338b-bcc7-6f5f4cc0b6c6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,6cbbeeb2-9f3a-00f6-2342-17139d6e97ae,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,801c95e3-b07e-7bfe-3789-a561c7beccd3,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d00f4ba6-707e-4684-20af-69eb47587cc2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,d8d4c0ea-3162-fa11-31e6-26675da44a38,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:tableau,dfe2c02a-54b7-f7a2-39fc-c651da2f6ad8,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity10,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity11,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity6,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.activity7,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:marketo-marketo,marketo.campaignstable,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.cmdb_ci,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.incident,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.problem,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_cat_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_req_item,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sc_request,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.sys_user_group,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:webdata-direct:servicenowitsm-servicenowitsm,ven01911.task,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:ATTRIBUTE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "ATTRIBUTE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:BINFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "BINFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:CALCULATEDFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "CALCULATEDFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COLUMNFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COLUMNFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:COUNT", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "COUNT" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DATASOURCEFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DATASOURCEFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:DIMENSION", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "DIMENSION" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:GROUPFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "GROUPFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:HIERARCHYFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "HIERARCHYFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:MEASURE", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "MEASURE" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SETFIELD", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SETFIELD" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:SUM", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "SUM" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:TagSheet3", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "TagSheet3" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:YEAR", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "YEAR" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:tag on published datasource", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "tag on published datasource" + } + }, + "systemMetadata": { + "lastObserved": 1638860400000, + "runId": "tableau-test", + "lastRunId": "no-run-id-provided", + "pipelineName": "test_tableau_no_hidden_assets_ingest" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json index cfc66af6f45699..044bc7b6d1e0b0 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_permission_ingestion_mces_golden.json @@ -943,6 +943,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1225,6 +1230,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1793,6 +1803,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2413,6 +2428,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2929,6 +2949,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3081,6 +3106,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3608,6 +3638,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4116,6 +4151,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4592,6 +4632,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5097,6 +5142,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5489,6 +5539,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5855,6 +5910,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6250,6 +6310,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6697,6 +6762,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7115,6 +7185,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7455,6 +7530,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7821,6 +7901,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8083,6 +8168,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8530,6 +8620,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8873,6 +8968,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9242,6 +9342,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9634,6 +9739,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10081,6 +10191,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10268,6 +10383,7 @@ "urn:li:chart:(tableau,f4317efd-c3e6-6ace-8fe6-e71b590bbbcc)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 1640200234000, @@ -10281,6 +10397,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10375,6 +10496,7 @@ "urn:li:chart:(tableau,8a6a269a-d6de-fae4-5050-513255b40ffc)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 1639773866000, @@ -10388,6 +10510,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10480,6 +10607,7 @@ "description": "", "charts": [], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 1639773866000, @@ -10493,6 +10621,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10599,6 +10732,7 @@ "urn:li:chart:(tableau,f76d3570-23b8-f74b-d85c-cc5484c2079c)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 1639768450000, @@ -10612,6 +10746,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11042,6 +11181,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -12900,7 +13044,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" }, { @@ -12908,7 +13052,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", "type": "TRANSFORMED" } ], @@ -12999,6 +13143,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13340,6 +13489,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14255,6 +14409,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15216,6 +15375,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22088,6 +22252,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26208,6 +26377,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31562,6 +31736,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31906,6 +32085,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32675,6 +32859,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json index 53d4a802541c35..621dbc642f6b1d 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_signout_timeout_mces_golden.json @@ -870,8 +870,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -939,6 +939,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1221,6 +1226,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1789,6 +1799,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2409,6 +2424,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2925,6 +2945,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3077,6 +3102,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3604,6 +3634,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4112,6 +4147,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4588,6 +4628,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5093,6 +5138,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5485,6 +5535,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5851,6 +5906,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6246,6 +6306,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6693,6 +6758,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7111,6 +7181,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7451,6 +7526,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7817,6 +7897,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8079,6 +8164,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8526,6 +8616,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8869,6 +8964,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9238,6 +9338,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9630,6 +9735,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10077,6 +10187,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10227,8 +10342,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10277,6 +10392,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10384,6 +10504,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10489,6 +10614,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10608,6 +10738,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11038,6 +11173,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -12896,7 +13036,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", "type": "TRANSFORMED" }, { @@ -12904,7 +13044,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,10c6297d-0dbd-44f1-b1ba-458bea446513,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:tableau,22b0b4c3-6b85-713d-a161-5a87fdd78f40,PROD)", "type": "TRANSFORMED" } ], @@ -12995,6 +13135,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13336,6 +13481,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14251,6 +14401,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15212,6 +15367,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22084,6 +22244,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26204,6 +26369,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31558,6 +31728,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31902,6 +32077,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32671,6 +32851,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json index 4811e462acdb2e..d5f3e86c86f16a 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_site_name_pattern_mces_golden.json @@ -1043,8 +1043,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -1112,6 +1112,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1398,6 +1403,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1970,6 +1980,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2594,6 +2609,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3114,6 +3134,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3270,6 +3295,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3805,6 +3835,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4317,6 +4352,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4797,6 +4837,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5306,6 +5351,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5702,6 +5752,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6072,6 +6127,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6471,6 +6531,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6922,6 +6987,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7344,6 +7414,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7688,6 +7763,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8058,6 +8138,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8324,6 +8409,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8775,6 +8865,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9122,6 +9217,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9495,6 +9595,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9891,6 +9996,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10342,6 +10452,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10496,8 +10611,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10546,6 +10661,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10657,6 +10777,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10766,6 +10891,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10889,6 +11019,11 @@ "dashboardUrl": "https://do-not-connect/#/site/site2/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11323,6 +11458,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13284,6 +13424,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13629,6 +13774,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14548,6 +14698,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15513,6 +15668,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22389,6 +22549,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26513,6 +26678,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31871,6 +32041,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32219,6 +32394,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32996,6 +33176,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json index 9c83fdbc1d8798..b4dfc7b4885c76 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_sites_as_container_mces_golden.json @@ -1043,8 +1043,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -1112,6 +1112,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1398,6 +1403,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1970,6 +1980,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2594,6 +2609,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3114,6 +3134,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3270,6 +3295,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3805,6 +3835,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4317,6 +4352,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4797,6 +4837,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5306,6 +5351,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5702,6 +5752,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6072,6 +6127,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6471,6 +6531,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6922,6 +6987,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7344,6 +7414,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7688,6 +7763,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8058,6 +8138,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8324,6 +8409,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8775,6 +8865,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9122,6 +9217,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9495,6 +9595,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9891,6 +9996,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10342,6 +10452,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10496,8 +10611,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10546,6 +10661,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10657,6 +10777,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10766,6 +10891,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10889,6 +11019,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11323,6 +11458,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13284,6 +13424,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13629,6 +13774,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14548,6 +14698,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15513,6 +15668,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22389,6 +22549,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26513,6 +26678,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31871,6 +32041,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32219,6 +32394,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32996,6 +33176,11 @@ "platform": "urn:li:dataPlatform:tableau" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json index fdecf887b53056..19ce71da97572f 100644 --- a/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/tableau/tableau_with_platform_instance_mces_golden.json @@ -817,8 +817,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 5 } @@ -887,6 +887,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1174,6 +1179,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -1747,6 +1757,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2372,6 +2387,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -2893,6 +2913,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3050,6 +3075,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -3587,6 +3617,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4100,6 +4135,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -4581,6 +4621,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5091,6 +5136,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5488,6 +5538,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -5859,6 +5914,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6259,6 +6319,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -6711,6 +6776,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7134,6 +7204,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7479,6 +7554,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -7850,6 +7930,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8117,6 +8202,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8569,6 +8659,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -8917,6 +9012,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9291,6 +9391,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -9688,6 +9793,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10140,6 +10250,11 @@ "actor": "urn:li:corpuser:unknown" } } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } } ] } @@ -10294,8 +10409,8 @@ "json": { "timestampMillis": 1638860400000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "viewsCount": 3 } @@ -10345,6 +10460,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/EmailPerformancebyCampaign/EmailPerformancebyCampaign" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10457,6 +10577,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/dvdRentalDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10567,6 +10692,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/dvdrental/Story1" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -10691,6 +10821,11 @@ "dashboardUrl": "https://do-not-connect/#/site/acryl/views/ExecutiveDashboard/ExecutiveDashboard" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -11126,6 +11261,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13088,6 +13228,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -13434,6 +13579,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -14354,6 +14504,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -15320,6 +15475,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -22197,6 +22357,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -26322,6 +26487,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -31681,6 +31851,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32030,6 +32205,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ @@ -32809,6 +32989,11 @@ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:tableau,acryl_site1)" } }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [] + } + }, { "com.linkedin.pegasus2avro.common.BrowsePaths": { "paths": [ diff --git a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py index 38a53b323876d1..1665b1401a636a 100644 --- a/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py +++ b/metadata-ingestion/tests/integration/tableau/test_tableau_ingest.py @@ -21,7 +21,7 @@ from datahub.emitter.mce_builder import DEFAULT_ENV, make_schema_field_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.ingestion.run.pipeline import Pipeline, PipelineContext +from datahub.ingestion.run.pipeline import Pipeline, PipelineContext, PipelineInitError from datahub.ingestion.source.tableau.tableau import ( TableauConfig, TableauSiteSource, @@ -579,7 +579,10 @@ def test_value_error_projects_and_project_pattern( new_config["projects"] = ["default"] new_config["project_pattern"] = {"allow": ["^Samples$"]} - try: + with pytest.raises( + PipelineInitError, + match=r".*projects is deprecated. Please use project_path_pattern only.*", + ): tableau_ingest_common( pytestconfig, tmp_path, @@ -589,8 +592,6 @@ def test_value_error_projects_and_project_pattern( mock_datahub_graph, pipeline_config=new_config, ) - except Exception as e: - assert "projects is deprecated. Please use project_path_pattern only" in str(e) def test_project_pattern_deprecation(pytestconfig, tmp_path, mock_datahub_graph): @@ -603,7 +604,10 @@ def test_project_pattern_deprecation(pytestconfig, tmp_path, mock_datahub_graph) new_config["project_pattern"] = {"allow": ["^Samples$"]} new_config["project_path_pattern"] = {"allow": ["^Samples$"]} - try: + with pytest.raises( + PipelineInitError, + match=r".*project_pattern is deprecated. Please use project_path_pattern only*", + ): tableau_ingest_common( pytestconfig, tmp_path, @@ -613,11 +617,6 @@ def test_project_pattern_deprecation(pytestconfig, tmp_path, mock_datahub_graph) mock_datahub_graph, pipeline_config=new_config, ) - except Exception as e: - assert ( - "project_pattern is deprecated. Please use project_path_pattern only" - in str(e) - ) def test_project_path_pattern_allow(pytestconfig, tmp_path, mock_datahub_graph): @@ -1225,6 +1224,100 @@ def test_permission_ingestion(pytestconfig, tmp_path, mock_datahub_graph): ) +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_no_hidden_assets(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_no_hidden_assets_mces.json" + golden_file_name: str = "tableau_no_hidden_assets_mces_golden.json" + + new_config = config_source_default.copy() + del new_config["projects"] + new_config["ingest_hidden_assets"] = False + + tableau_ingest_common( + pytestconfig, + tmp_path, + mock_data(), + golden_file_name, + output_file_name, + mock_datahub_graph, + pipeline_config=new_config, + pipeline_name="test_tableau_no_hidden_assets_ingest", + ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_ingest_tags_disabled(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_ingest_tags_disabled_mces.json" + golden_file_name: str = "tableau_ingest_tags_disabled_mces_golden.json" + + new_config = config_source_default.copy() + new_config["ingest_tags"] = False + + tableau_ingest_common( + pytestconfig, + tmp_path, + mock_data(), + golden_file_name, + output_file_name, + mock_datahub_graph, + pipeline_config=new_config, + pipeline_name="test_tableau_ingest_tags_disabled", + ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_hidden_asset_tags(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_hidden_asset_tags_mces.json" + golden_file_name: str = "tableau_hidden_asset_tags_mces_golden.json" + + new_config = config_source_default.copy() + del new_config["projects"] + new_config["tags_for_hidden_assets"] = ["hidden", "private"] + + tableau_ingest_common( + pytestconfig, + tmp_path, + mock_data(), + golden_file_name, + output_file_name, + mock_datahub_graph, + pipeline_config=new_config, + pipeline_name="test_tableau_hidden_asset_tags_ingest", + ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_hidden_assets_without_ingest_tags(pytestconfig, tmp_path, mock_datahub_graph): + enable_logging() + output_file_name: str = "tableau_hidden_asset_tags_error_mces.json" + golden_file_name: str = "tableau_hidden_asset_tags_error_mces_golden.json" + + new_config = config_source_default.copy() + new_config["tags_for_hidden_assets"] = ["hidden", "private"] + new_config["ingest_tags"] = False + + with pytest.raises( + PipelineInitError, + match=r".*tags_for_hidden_assets is only allowed with ingest_tags enabled.*", + ): + tableau_ingest_common( + pytestconfig, + tmp_path, + mock_data(), + golden_file_name, + output_file_name, + mock_datahub_graph, + pipeline_config=new_config, + ) + + @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_permission_mode_switched_error(pytestconfig, tmp_path, mock_datahub_graph): From 2babbe6b0f1e48ca8d66e3c082816de2247e6028 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 9 Dec 2024 14:08:49 -0500 Subject: [PATCH 153/174] feat(airflow): drop Airflow < 2.3 support + make plugin v2 the default (#12056) --- .github/workflows/airflow-plugin.yml | 16 +- docs/how/updating-datahub.md | 2 + docs/lineage/airflow.md | 24 +-- .../airflow-plugin/build.gradle | 2 +- .../airflow-plugin/setup.py | 14 +- .../datahub_airflow_plugin/_airflow_shims.py | 4 +- .../datahub_listener.py | 3 +- .../integration/goldens/v1_basic_iolets.json | 54 ++++++- .../integration/goldens/v1_simple_dag.json | 142 ++++++++++++++++-- .../tests/integration/test_plugin.py | 24 ++- .../airflow-plugin/tox.ini | 28 ++-- 11 files changed, 241 insertions(+), 72 deletions(-) diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index 66a08dc63aa0de..1fdfc52857b011 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -34,29 +34,21 @@ jobs: include: # Note: this should be kept in sync with tox.ini. - python-version: "3.8" - extra_pip_requirements: "apache-airflow~=2.1.4" - extra_pip_extras: plugin-v1 - - python-version: "3.8" - extra_pip_requirements: "apache-airflow~=2.2.4" - extra_pip_extras: plugin-v1 + extra_pip_requirements: "apache-airflow~=2.3.4" + extra_pip_extras: test-airflow23 - python-version: "3.10" extra_pip_requirements: "apache-airflow~=2.4.3" - extra_pip_extras: plugin-v2,test-airflow24 + extra_pip_extras: test-airflow24 - python-version: "3.10" extra_pip_requirements: "apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt" - extra_pip_extras: plugin-v2 - python-version: "3.10" extra_pip_requirements: "apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt" - extra_pip_extras: plugin-v2 - python-version: "3.10" extra_pip_requirements: "apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt" - extra_pip_extras: plugin-v2 - python-version: "3.11" extra_pip_requirements: "apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt" - extra_pip_extras: plugin-v2 - python-version: "3.11" - extra_pip_requirements: "apache-airflow~=2.10.2 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.11.txt" - extra_pip_extras: plugin-v2 + extra_pip_requirements: "apache-airflow~=2.10.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt" fail-fast: false steps: - name: Set up JDK 17 diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 087e30c2e541ad..3f9de1ff2f7f9e 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -35,6 +35,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11701: The Fivetran `sources_to_database` field is deprecated in favor of setting directly within `sources_to_platform_instance..database`. - #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information. +- #12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2. +- #12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation. ### Breaking Changes diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index 829c048a8f8e24..2bd58334933fb7 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -13,14 +13,14 @@ The DataHub Airflow plugin supports: - Task run information, including task successes and failures. - Manual lineage annotations using `inlets` and `outlets` on Airflow operators. -There's two actively supported implementations of the plugin, with different Airflow version support. +There's two implementations of the plugin, with different Airflow version support. -| Approach | Airflow Version | Notes | -| --------- | --------------- | --------------------------------------------------------------------------- | -| Plugin v2 | 2.3.4+ | Recommended. Requires Python 3.8+ | -| Plugin v1 | 2.1 - 2.8 | No automatic lineage extraction; may not extract lineage if the task fails. | +| Approach | Airflow Versions | Notes | +| --------- | ---------------- | --------------------------------------------------------------------------------------- | +| Plugin v2 | 2.3.4+ | Recommended. Requires Python 3.8+ | +| Plugin v1 | 2.3 - 2.8 | Deprecated. No automatic lineage extraction; may not extract lineage if the task fails. | -If you're using Airflow older than 2.1, it's possible to use the v1 plugin with older versions of `acryl-datahub-airflow-plugin`. See the [compatibility section](#compatibility) for more details. +If you're using Airflow older than 2.3, it's possible to use the v1 plugin with older versions of `acryl-datahub-airflow-plugin`. See the [compatibility section](#compatibility) for more details. @@ -29,7 +29,7 @@ If you're using Airflow older than 2.1, it's possible to use the v1 plugin with ### Installation -The v2 plugin requires Airflow 2.3+ and Python 3.8+. If you don't meet these requirements, use the v1 plugin instead. +The v2 plugin requires Airflow 2.3+ and Python 3.8+. If you don't meet these requirements, see the [compatibility section](#compatibility) for other options. ```shell pip install 'acryl-datahub-airflow-plugin[plugin-v2]' @@ -84,9 +84,10 @@ enabled = True # default ### Installation -The v1 plugin requires Airflow 2.1 - 2.8 and Python 3.8+. If you're on older versions, it's still possible to use an older version of the plugin. See the [compatibility section](#compatibility) for more details. +The v1 plugin requires Airflow 2.3 - 2.8 and Python 3.8+. If you're on older versions, it's still possible to use an older version of the plugin. See the [compatibility section](#compatibility) for more details. -If you're using Airflow 2.3+, we recommend using the v2 plugin instead. If you need to use the v1 plugin with Airflow 2.3+, you must also set the environment variable `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN=true`. +Note that the v1 plugin is less featureful than the v2 plugin, and is overall not actively maintained. +Since datahub v0.15.0, the v2 plugin has been the default. If you need to use the v1 plugin with `acryl-datahub-airflow-plugin` v0.15.0+, you must also set the environment variable `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN=true`. ```shell pip install 'acryl-datahub-airflow-plugin[plugin-v1]' @@ -340,11 +341,12 @@ The solution is to upgrade `acryl-datahub-airflow-plugin>=0.12.0.4` or upgrade ` ## Compatibility -We no longer officially support Airflow <2.1. However, you can use older versions of `acryl-datahub-airflow-plugin` with older versions of Airflow. -Both of these options support Python 3.7+. +We no longer officially support Airflow <2.3. However, you can use older versions of `acryl-datahub-airflow-plugin` with older versions of Airflow. +The first two options support Python 3.7+, and the last option supports Python 3.8+. - Airflow 1.10.x, use DataHub plugin v1 with acryl-datahub-airflow-plugin <= 0.9.1.0. - Airflow 2.0.x, use DataHub plugin v1 with acryl-datahub-airflow-plugin <= 0.11.0.1. +- Airflow 2.2.x, use DataHub plugin v2 with acryl-datahub-airflow-plugin <= 0.14.1.5. DataHub also previously supported an Airflow [lineage backend](https://airflow.apache.org/docs/apache-airflow/2.2.0/lineage.html#lineage-backend) implementation. While the implementation is still in our codebase, it is deprecated and will be removed in a future release. Note that the lineage backend did not support automatic lineage extraction, did not capture task failures, and did not work in AWS MWAA. diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index f30858ba6a14ef..68a35c0dfc417b 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -13,7 +13,7 @@ if (!project.hasProperty("extra_pip_requirements")) { ext.extra_pip_requirements = "" } if (!project.hasProperty("extra_pip_extras")) { - ext.extra_pip_extras = "plugin-v2" + ext.extra_pip_extras = "" } // If extra_pip_extras is non-empty, we need to add a comma to the beginning of the string. if (extra_pip_extras != "") { diff --git a/metadata-ingestion-modules/airflow-plugin/setup.py b/metadata-ingestion-modules/airflow-plugin/setup.py index 02a0bbb6022e04..3209233184d55a 100644 --- a/metadata-ingestion-modules/airflow-plugin/setup.py +++ b/metadata-ingestion-modules/airflow-plugin/setup.py @@ -24,8 +24,8 @@ def get_long_description(): base_requirements = { f"acryl-datahub[datahub-rest]{_self_pin}", - # Actual dependencies. - "apache-airflow >= 2.0.2", + # We require Airflow 2.3.x, since we need the new DAG listener API. + "apache-airflow>=2.3.0", } plugins: Dict[str, Set[str]] = { @@ -44,12 +44,13 @@ def get_long_description(): # We remain restrictive on the versions allowed here to prevent # us from being broken by backwards-incompatible changes in the # underlying package. - "openlineage-airflow>=1.2.0,<=1.22.0", + "openlineage-airflow>=1.2.0,<=1.25.0", }, } -# Include datahub-rest in the base requirements. +# Require some plugins by default. base_requirements.update(plugins["datahub-rest"]) +base_requirements.update(plugins["plugin-v2"]) mypy_stubs = { @@ -109,6 +110,11 @@ def get_long_description(): "apache-airflow-providers-sqlite", } per_version_test_requirements = { + "test-airflow23": { + "pendulum<3.0", + "Flask-Session<0.6.0", + "connexion<3.0", + }, "test-airflow24": { "pendulum<3.0", "Flask-Session<0.6.0", diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py index c1e2dd4cc422d0..d86a46e042e8f8 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_airflow_shims.py @@ -46,7 +46,7 @@ def get_task_inlets(operator: "Operator") -> List: return operator._inlets # type: ignore[attr-defined, union-attr] if hasattr(operator, "get_inlet_defs"): return operator.get_inlet_defs() # type: ignore[attr-defined] - return operator.inlets + return operator.inlets or [] def get_task_outlets(operator: "Operator") -> List: @@ -56,7 +56,7 @@ def get_task_outlets(operator: "Operator") -> List: return operator._outlets # type: ignore[attr-defined, union-attr] if hasattr(operator, "get_outlet_defs"): return operator.get_outlet_defs() - return operator.outlets + return operator.outlets or [] __all__ = [ diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py index e00cf51ea456cc..aa7b3108f64f1e 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py @@ -74,7 +74,7 @@ def hookimpl(f: _F) -> _F: # type: ignore[misc] # noqa: F811 "1", ) _RUN_IN_THREAD_TIMEOUT = float( - os.getenv("DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD_TIMEOUT", 15) + os.getenv("DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD_TIMEOUT", 10) ) _DATAHUB_CLEANUP_DAG = "Datahub_Cleanup" @@ -102,6 +102,7 @@ def get_airflow_plugin_listener() -> Optional["DataHubListener"]: "capture_tags": plugin_config.capture_tags_info, "capture_ownership": plugin_config.capture_ownership_info, "enable_extractors": plugin_config.enable_extractors, + "render_templates": plugin_config.render_templates, "disable_openlineage_plugin": plugin_config.disable_openlineage_plugin, }, ) diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json index fd10f858d00fb6..4c21b7ed4000dc 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_basic_iolets.json @@ -14,7 +14,7 @@ "fileloc": "", "is_paused_upon_creation": "None", "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))", - "tags": "None", + "tags": "[]", "timezone": "Timezone('UTC')" }, "externalUrl": "http://airflow.example.com/tree?dag_id=basic_iolets", @@ -83,7 +83,7 @@ "execution_timeout": "None", "sla": "None", "task_id": "'run_data_task'", - "trigger_rule": "'all_success'", + "trigger_rule": "", "wait_for_downstream": "False", "downstream_task_ids": "[]", "inlets": "[]", @@ -246,6 +246,46 @@ } } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'run_data_task'", + "execution_timeout": "None", + "sla": "None", + "task_id": "'run_data_task'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=basic_iolets&_flt_3_task_id=run_data_task", + "name": "run_data_task", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,basic_iolets,prod),run_data_task)", @@ -402,16 +442,16 @@ "state": "success", "operator": "BashOperator", "priority_weight": "1", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets", + "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1", "orchestrator": "airflow", "dag_id": "basic_iolets", "task_id": "run_data_task" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets", + "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_data_task&dag_id=basic_iolets&map_index=-1", "name": "basic_iolets_run_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1717180290951, + "time": 1733529136396, "actor": "urn:li:corpuser:datahub" } } @@ -544,7 +584,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1717180290951, + "timestampMillis": 1733529136396, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -561,7 +601,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1717180291140, + "timestampMillis": 1733529137385, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json index 5c5be6848fd839..b6ab1ff9120f25 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens/v1_simple_dag.json @@ -14,7 +14,7 @@ "fileloc": "", "is_paused_upon_creation": "None", "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))", - "tags": "None", + "tags": "[]", "timezone": "Timezone('UTC')" }, "externalUrl": "http://airflow.example.com/tree?dag_id=simple_dag", @@ -84,7 +84,7 @@ "execution_timeout": "None", "sla": "None", "task_id": "'task_1'", - "trigger_rule": "'all_success'", + "trigger_rule": "", "wait_for_downstream": "False", "downstream_task_ids": "['run_another_data_task']", "inlets": "[]", @@ -205,6 +205,46 @@ } } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'task_1'", + "execution_timeout": "None", + "sla": "None", + "task_id": "'task_1'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "['run_another_data_task']", + "inlets": "[]", + "outlets": "[]" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=task_1", + "name": "task_1", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),task_1)", @@ -319,16 +359,16 @@ "state": "success", "operator": "BashOperator", "priority_weight": "2", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag", + "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1", "orchestrator": "airflow", "dag_id": "simple_dag", "task_id": "task_1" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag", + "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=task_1&dag_id=simple_dag&map_index=-1", "name": "simple_dag_task_1_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1717180227827, + "time": 1733528983395, "actor": "urn:li:corpuser:datahub" } } @@ -419,7 +459,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1717180227827, + "timestampMillis": 1733528983395, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -436,7 +476,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1717180228022, + "timestampMillis": 1733528984355, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -449,6 +489,42 @@ } } }, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)", + "changeType": "UPSERT", + "aspectName": "dataFlowInfo", + "aspect": { + "json": { + "customProperties": { + "_access_control": "None", + "catchup": "False", + "description": "'A simple DAG that runs a few fake data tasks.'", + "doc_md": "None", + "fileloc": "", + "is_paused_upon_creation": "None", + "start_date": "DateTime(2023, 1, 1, 0, 0, 0, tzinfo=Timezone('UTC'))", + "tags": "[]", + "timezone": "Timezone('UTC')" + }, + "externalUrl": "http://airflow.example.com/tree?dag_id=simple_dag", + "name": "simple_dag", + "description": "A simple DAG that runs a few fake data tasks.", + "env": "PROD" + } + } +}, +{ + "entityType": "dataFlow", + "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataFlow", "entityUrn": "urn:li:dataFlow:(airflow,simple_dag,prod)", @@ -498,7 +574,7 @@ "execution_timeout": "None", "sla": "None", "task_id": "'run_another_data_task'", - "trigger_rule": "'all_success'", + "trigger_rule": "", "wait_for_downstream": "False", "downstream_task_ids": "[]", "inlets": "[]", @@ -575,6 +651,46 @@ } } }, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)", + "changeType": "UPSERT", + "aspectName": "dataJobInfo", + "aspect": { + "json": { + "customProperties": { + "depends_on_past": "False", + "email": "None", + "label": "'run_another_data_task'", + "execution_timeout": "None", + "sla": "None", + "task_id": "'run_another_data_task'", + "trigger_rule": "", + "wait_for_downstream": "False", + "downstream_task_ids": "[]", + "inlets": "[]", + "outlets": "[]" + }, + "externalUrl": "http://airflow.example.com/taskinstance/list/?flt1_dag_id_equals=simple_dag&_flt_3_task_id=run_another_data_task", + "name": "run_another_data_task", + "type": { + "string": "COMMAND" + }, + "env": "PROD" + } + } +}, +{ + "entityType": "dataJob", + "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + } +}, { "entityType": "dataJob", "entityUrn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,simple_dag,prod),run_another_data_task)", @@ -645,16 +761,16 @@ "state": "success", "operator": "BashOperator", "priority_weight": "1", - "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag", + "log_url": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1", "orchestrator": "airflow", "dag_id": "simple_dag", "task_id": "run_another_data_task" }, - "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag", + "externalUrl": "http://airflow.example.com/log?execution_date=2023-09-27T21%3A34%3A38%2B00%3A00&task_id=run_another_data_task&dag_id=simple_dag&map_index=-1", "name": "simple_dag_run_another_data_task_manual_run_test", "type": "BATCH_AD_HOC", "created": { - "time": 1717180231676, + "time": 1733528992448, "actor": "urn:li:corpuser:datahub" } } @@ -679,7 +795,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1717180231676, + "timestampMillis": 1733528992448, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" @@ -696,7 +812,7 @@ "aspectName": "dataProcessInstanceRunEvent", "aspect": { "json": { - "timestampMillis": 1717180231824, + "timestampMillis": 1733528993380, "partitionSpec": { "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py index 75bb43af1a43dd..3b2c9140e4632f 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py @@ -12,6 +12,7 @@ import time from typing import Any, Iterator, Sequence +import packaging.version import pytest import requests import tenacity @@ -20,6 +21,7 @@ from datahub.testing.compare_metadata_json import assert_metadata_files_equal from datahub_airflow_plugin._airflow_shims import ( + AIRFLOW_VERSION, HAS_AIRFLOW_DAG_LISTENER_API, HAS_AIRFLOW_LISTENER_API, HAS_AIRFLOW_STANDALONE_CMD, @@ -242,6 +244,7 @@ def _run_airflow( # Note that we could also disable the RUN_IN_THREAD entirely, # but I want to minimize the difference between CI and prod. "DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD_TIMEOUT": "30", + "DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN": "true" if is_v1 else "false", # Convenience settings. "AIRFLOW__DATAHUB__LOG_LEVEL": "DEBUG", "AIRFLOW__DATAHUB__DEBUG_EMITTER": "True", @@ -361,7 +364,6 @@ class DagTestCase: @pytest.mark.parametrize( ["golden_filename", "test_case", "is_v1"], [ - # On Airflow <= 2.2, test plugin v1. *[ pytest.param( f"v1_{test_case.dag_id}", @@ -369,8 +371,8 @@ class DagTestCase: True, id=f"v1_{test_case.dag_id}", marks=pytest.mark.skipif( - HAS_AIRFLOW_LISTENER_API, - reason="Not testing plugin v1 on newer Airflow versions", + AIRFLOW_VERSION >= packaging.version.parse("2.4.0"), + reason="We only test the v1 plugin on Airflow 2.3", ), ) for test_case in test_cases @@ -391,10 +393,18 @@ class DagTestCase: if HAS_AIRFLOW_DAG_LISTENER_API else f"v2_{test_case.dag_id}_no_dag_listener" ), - marks=pytest.mark.skipif( - not HAS_AIRFLOW_LISTENER_API, - reason="Cannot test plugin v2 without the Airflow plugin listener API", - ), + marks=[ + pytest.mark.skipif( + not HAS_AIRFLOW_LISTENER_API, + reason="Cannot test plugin v2 without the Airflow plugin listener API", + ), + pytest.mark.skipif( + AIRFLOW_VERSION < packaging.version.parse("2.4.0"), + reason="We skip testing the v2 plugin on Airflow 2.3 because it causes flakiness in the custom properties. " + "Ideally we'd just fix these, but given that Airflow 2.3 is EOL and likely going to be deprecated " + "soon anyways, it's not worth the effort.", + ), + ], ) for test_case in test_cases ], diff --git a/metadata-ingestion-modules/airflow-plugin/tox.ini b/metadata-ingestion-modules/airflow-plugin/tox.ini index 28c0b9532bcb8e..b310ec84248f17 100644 --- a/metadata-ingestion-modules/airflow-plugin/tox.ini +++ b/metadata-ingestion-modules/airflow-plugin/tox.ini @@ -4,17 +4,24 @@ # and then run "tox" from this directory. [tox] -envlist = py38-airflow21, py38-airflow22, py310-airflow24, py310-airflow26, py310-airflow27, py310-airflow28, py311-airflow29, py311-airflow210 +envlist = py38-airflow23, py310-airflow24, py310-airflow26, py310-airflow27, py310-airflow28, py311-airflow29, py311-airflow210 [testenv] use_develop = true -extras = dev,integration-tests,plugin-v1 +extras = + dev + integration-tests + plugin-v1 + plugin-v2 + # For Airflow 2.3 and 2.4, add a few extra requirements. + airflow23: test-airflow23 + airflow24: test-airflow24 + deps = # This should be kept in sync with the Github Actions matrix. -e ../../metadata-ingestion/ # Airflow version - airflow21: apache-airflow~=2.1.0 - airflow22: apache-airflow~=2.2.0 + airflow23: apache-airflow~=2.3.0 airflow24: apache-airflow~=2.4.0 airflow26: apache-airflow~=2.6.0 airflow27: apache-airflow~=2.7.0 @@ -23,7 +30,8 @@ deps = airflow210: apache-airflow~=2.10.0 # Respect the Airflow constraints files. - # We can't make ourselves work with the constraints of Airflow < 2.3. + # We can't make ourselves work with the constraints of Airflow <= 2.3. + ; py38-airflow23: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.3.4/constraints-3.8.txt # The Airflow 2.4 constraints file requires a version of the sqlite provider whose # hook type is missing the `conn_name_attr` property. ; py310-airflow24: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.4.3/constraints-3.10.txt @@ -31,7 +39,7 @@ deps = py310-airflow27: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt py310-airflow28: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt py311-airflow29: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt - py311-airflow210: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.11.txt + py311-airflow210: -c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt # Before pinning to the constraint files, we previously left the dependencies # more open. There were a number of packages for which this caused issues. @@ -54,11 +62,3 @@ deps = ; airflow24,airflow26,airflow27,airflow28: Flask-Session<0.6.0 commands = pytest --cov-append {posargs} - -# For Airflow 2.4+, add the plugin-v2 extra. -[testenv:py310-airflow24] -extras = dev,integration-tests,plugin-v2,test-airflow24 - -[testenv:py3{10,11}-airflow{26,27,28,29,210}] -extras = dev,integration-tests,plugin-v2 - From bb67af03c5147b53df5fee1511d968091de28718 Mon Sep 17 00:00:00 2001 From: Jay <159848059+jayacryl@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:17:18 -0500 Subject: [PATCH 154/174] fix(web) disallow deselecting all degrees on impact analysis view (#12063) --- .../src/app/search/SimpleSearchFilters.tsx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx index 416b04403723f5..877efb55fcf828 100644 --- a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx +++ b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx @@ -4,7 +4,12 @@ import { FacetFilterInput, FacetMetadata } from '../../types.generated'; import { FilterScenarioType } from './filters/render/types'; import { useFilterRendererRegistry } from './filters/render/useFilterRenderer'; import { SimpleSearchFilter } from './SimpleSearchFilter'; -import { ENTITY_FILTER_NAME, ENTITY_INDEX_FILTER_NAME, LEGACY_ENTITY_FILTER_NAME } from './utils/constants'; +import { + DEGREE_FILTER_NAME, + ENTITY_FILTER_NAME, + ENTITY_INDEX_FILTER_NAME, + LEGACY_ENTITY_FILTER_NAME, +} from './utils/constants'; const TOP_FILTERS = ['degree', ENTITY_FILTER_NAME, 'platform', 'tags', 'glossaryTerms', 'domains', 'owners']; @@ -43,6 +48,15 @@ export const SimpleSearchFilters = ({ facets, selectedFilters, onFilterSelect, l : filter, ) .filter((filter) => filter.field !== field || !(filter.values?.length === 0)); + + // Do not let user unselect all degree filters + if (field === DEGREE_FILTER_NAME && !selected) { + const hasDegreeFilter = newFilters.find((filter) => filter.field === DEGREE_FILTER_NAME); + if (!hasDegreeFilter) { + return; + } + } + setCachedProps({ ...cachedProps, selectedFilters: newFilters }); onFilterSelect(newFilters); }; From 8a1c1804b767d4cb3f36bf76e79e799a4088a03c Mon Sep 17 00:00:00 2001 From: kanavnarula <33712591+kanavnarula@users.noreply.github.com> Date: Tue, 10 Dec 2024 02:33:09 +0530 Subject: [PATCH 155/174] feat(): Add parent container hierarchy label to the container (#11705) Co-authored-by: mac Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> Co-authored-by: Kanav Narula Co-authored-by: mac Co-authored-by: mac --- .../sidebar/Container/ContainerSelectModal.tsx | 14 +++++++++++++- .../src/app/search/SearchFilterLabel.tsx | 11 ++++++++++- .../src/app/search/filters/FilterOption.tsx | 3 ++- datahub-web-react/src/app/search/filters/utils.tsx | 14 ++++++++++++++ datahub-web-react/src/graphql/fragments.graphql | 1 + datahub-web-react/src/graphql/search.graphql | 3 +++ 6 files changed, 43 insertions(+), 3 deletions(-) diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx index 681f89831b92c4..818d75c37696dc 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Container/ContainerSelectModal.tsx @@ -5,6 +5,8 @@ import { useGetSearchResultsLazyQuery } from '../../../../../../../graphql/searc import { Container, Entity, EntityType } from '../../../../../../../types.generated'; import { useEnterKeyListener } from '../../../../../../shared/useEnterKeyListener'; import { useEntityRegistry } from '../../../../../../useEntityRegistry'; +import { getParentEntities } from '../../../../../../search/filters/utils'; +import ParentEntities from '../../../../../../search/filters/ParentEntities'; type Props = { onCloseModal: () => void; @@ -26,7 +28,7 @@ const StyleTag = styled(Tag)` align-items: center; `; -const PreviewImage = styled.img` +export const PreviewImage = styled.img` max-height: 18px; width: auto; object-fit: contain; @@ -34,6 +36,10 @@ const PreviewImage = styled.img` margin-right: 4px; `; +export const ParentWrapper = styled.div` + max-width: 400px; +`; + export const ContainerSelectModal = ({ onCloseModal, defaultValues, onOkOverride, titleOverride }: Props) => { const [containerSearch, { data: platforSearchData }] = useGetSearchResultsLazyQuery(); const entityRegistry = useEntityRegistry(); @@ -65,10 +71,16 @@ export const ContainerSelectModal = ({ onCloseModal, defaultValues, onOkOverride // Renders a search result in the select dropdown. const renderSearchResult = (entity: Container) => { const displayName = entityRegistry.getDisplayName(EntityType.Container, entity); + const parentEntities: Entity[] = getParentEntities(entity as Entity) || []; const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( + {parentEntities.length > 0 && ( + + + + )} {truncatedDisplayName} diff --git a/datahub-web-react/src/app/search/SearchFilterLabel.tsx b/datahub-web-react/src/app/search/SearchFilterLabel.tsx index 5a0e75cc2ae1ce..2ed2d4608de0d2 100644 --- a/datahub-web-react/src/app/search/SearchFilterLabel.tsx +++ b/datahub-web-react/src/app/search/SearchFilterLabel.tsx @@ -23,6 +23,9 @@ import CustomAvatar from '../shared/avatar/CustomAvatar'; import { IconStyleType } from '../entity/Entity'; import { formatNumber } from '../shared/formatNumber'; import useGetBrowseV2LabelOverride from './filters/useGetBrowseV2LabelOverride'; +import { getParentEntities } from './filters/utils'; +import { ParentWrapper } from '../entity/shared/containers/profile/sidebar/Container/ContainerSelectModal'; +import ParentEntities from './filters/ParentEntities'; type Props = { field: string; @@ -157,11 +160,17 @@ export const SearchFilterLabel = ({ field, value, entity, count, hideCount }: Pr if (entity?.type === EntityType.Container) { const container = entity as Container; const displayName = entityRegistry.getDisplayName(EntityType.Container, container); + const parentEntities: Entity[] = getParentEntities(container as Entity) || []; const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( {!!container.platform?.properties?.logoUrl && ( - + <> + + + + + )} {truncatedDisplayName} diff --git a/datahub-web-react/src/app/search/filters/FilterOption.tsx b/datahub-web-react/src/app/search/filters/FilterOption.tsx index 3749f44cbf6718..50b78c7f0685c9 100644 --- a/datahub-web-react/src/app/search/filters/FilterOption.tsx +++ b/datahub-web-react/src/app/search/filters/FilterOption.tsx @@ -8,6 +8,7 @@ import { generateColor } from '../../entity/shared/components/styled/StyledTag'; import { ANTD_GRAY } from '../../entity/shared/constants'; import { useEntityRegistry } from '../../useEntityRegistry'; import { + CONTAINER_FILTER_NAME, ENTITY_SUB_TYPE_FILTER_NAME, MAX_COUNT_VAL, PLATFORM_FILTER_NAME, @@ -125,7 +126,7 @@ export default function FilterOption({ const { field, value, count, entity } = filterOption; const entityRegistry = useEntityRegistry(); const { icon, label } = getFilterIconAndLabel(field, value, entityRegistry, entity || null, 14); - const shouldShowIcon = field === PLATFORM_FILTER_NAME && icon !== null; + const shouldShowIcon = (field === PLATFORM_FILTER_NAME || field === CONTAINER_FILTER_NAME) && icon !== null; const shouldShowTagColor = field === TAGS_FILTER_NAME && entity?.type === EntityType.Tag; const isSubTypeFilter = field === TYPE_NAMES_FILTER_NAME; const parentEntities: Entity[] = getParentEntities(entity as Entity) || []; diff --git a/datahub-web-react/src/app/search/filters/utils.tsx b/datahub-web-react/src/app/search/filters/utils.tsx index f115277a049674..bd747777d11175 100644 --- a/datahub-web-react/src/app/search/filters/utils.tsx +++ b/datahub-web-react/src/app/search/filters/utils.tsx @@ -20,6 +20,7 @@ import { FacetFilterInput, FacetMetadata, GlossaryTerm, + Container, } from '../../../types.generated'; import { IconStyleType } from '../../entity/Entity'; import { @@ -186,6 +187,15 @@ export function getFilterIconAndLabel( entityRegistry.getIcon(EntityType.DataPlatform, size || 12, IconStyleType.ACCENT, ANTD_GRAY[9]) ); label = filterEntity ? entityRegistry.getDisplayName(EntityType.DataPlatform, filterEntity) : filterValue; + } else if (filterField === CONTAINER_FILTER_NAME) { + // Scenario where the filter entity exists and filterField is container + const logoUrl = (filterEntity as Container)?.platform?.properties?.logoUrl; + icon = logoUrl ? ( + + ) : ( + entityRegistry.getIcon(EntityType.DataPlatform, size || 12, IconStyleType.ACCENT, ANTD_GRAY[9]) + ); + label = entityRegistry.getDisplayName(EntityType.Container, filterEntity); } else if (filterField === BROWSE_PATH_V2_FILTER_NAME) { icon = ; label = getLastBrowseEntryFromFilterValue(filterValue); @@ -196,6 +206,7 @@ export function getFilterIconAndLabel( filterEntity, size, ); + icon = newIcon; label = newLabel; } else { @@ -344,6 +355,9 @@ export function getParentEntities(entity: Entity): Entity[] | null { if (entity.type === EntityType.Domain) { return (entity as Domain).parentDomains?.domains || []; } + if (entity.type === EntityType.Container) { + return (entity as Container).parentContainers?.containers || []; + } return null; } diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index 67dbdbbb22f309..ade63f151d1a09 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -1010,6 +1010,7 @@ fragment entityContainer on Container { fragment parentContainerFields on Container { urn + type properties { name } diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 4a10d5fe250de1..3e26dd7121b72c 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -910,6 +910,9 @@ fragment facetFields on FacetMetadata { properties { name } + parentContainers { + ...parentContainersFields + } } ... on CorpUser { username From e4ea993df17b6bee7d04c581fbe2bd9bbcf34927 Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Mon, 9 Dec 2024 16:40:31 -0800 Subject: [PATCH 156/174] fix(py-sdk): DataJobPatchBuilder handling timestamps, output edges (#12067) Co-authored-by: Harshal Sheth --- docs/how/updating-datahub.md | 9 +- .../src/datahub/specific/datajob.py | 14 +-- .../nifi/nifi_mces_golden_standalone.json | 40 +----- .../tests/unit/patch/test_patch_builder.py | 117 ++++++++++++------ .../tests/patch/test_datajob_patches.py | 94 ++++++++++++++ 5 files changed, 190 insertions(+), 84 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 3f9de1ff2f7f9e..bcc89332cc1c1b 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -48,7 +48,14 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11619 - schema field/column paths can no longer be duplicated within the schema - #11570 - The `DatahubClientConfig`'s server field no longer defaults to `http://localhost:8080`. Be sure to explicitly set this. - #11570 - If a `datahub_api` is explicitly passed to a stateful ingestion config provider, it will be used. We previously ignored it if the pipeline context also had a graph object. -- #11518 - DataHub Garbage Collection: Various entities that are soft-deleted (after 10d) or are timeseries *entities* (dataprocess, execution requests) will be removed automatically using logic in the `datahub-gc` ingestion source. +- #11518 - DataHub Garbage Collection: Various entities that are soft-deleted + (after 10d) or are timeseries *entities* (dataprocess, execution requests) + will be removed automatically using logic in the `datahub-gc` ingestion + source. +- #12067 - Default behavior of DataJobPatchBuilder in Python sdk has been + changed to NOT fill out `created` and `lastModified` auditstamps by default + for input and output dataset edges. This should not have any user-observable + impact (time-based lineage viz will still continue working based on observed time), but could break assumptions previously being made by clients. ### Potential Downtime diff --git a/metadata-ingestion/src/datahub/specific/datajob.py b/metadata-ingestion/src/datahub/specific/datajob.py index fb7b0ae7816f17..6ff4741b09c26a 100644 --- a/metadata-ingestion/src/datahub/specific/datajob.py +++ b/metadata-ingestion/src/datahub/specific/datajob.py @@ -102,7 +102,7 @@ def add_input_datajob(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde Notes: If `input` is an Edge object, it is used directly. If `input` is a Urn object or string, - it is converted to an Edge object and added with default audit stamps. + it is converted to an Edge object and added without any audit stamps. """ if isinstance(input, Edge): input_urn: str = input.destinationUrn @@ -114,8 +114,6 @@ def add_input_datajob(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde input_edge = Edge( destinationUrn=input_urn, - created=self._mint_auditstamp(), - lastModified=self._mint_auditstamp(), ) self._ensure_urn_type("dataJob", [input_edge], "add_input_datajob") @@ -185,7 +183,7 @@ def add_input_dataset(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde Notes: If `input` is an Edge object, it is used directly. If `input` is a Urn object or string, - it is converted to an Edge object and added with default audit stamps. + it is converted to an Edge object and added without any audit stamps. """ if isinstance(input, Edge): input_urn: str = input.destinationUrn @@ -197,8 +195,6 @@ def add_input_dataset(self, input: Union[Edge, Urn, str]) -> "DataJobPatchBuilde input_edge = Edge( destinationUrn=input_urn, - created=self._mint_auditstamp(), - lastModified=self._mint_auditstamp(), ) self._ensure_urn_type("dataset", [input_edge], "add_input_dataset") @@ -270,7 +266,7 @@ def add_output_dataset( Notes: If `output` is an Edge object, it is used directly. If `output` is a Urn object or string, - it is converted to an Edge object and added with default audit stamps. + it is converted to an Edge object and added without any audit stamps. """ if isinstance(output, Edge): output_urn: str = output.destinationUrn @@ -282,15 +278,13 @@ def add_output_dataset( output_edge = Edge( destinationUrn=output_urn, - created=self._mint_auditstamp(), - lastModified=self._mint_auditstamp(), ) self._ensure_urn_type("dataset", [output_edge], "add_output_dataset") self._add_patch( DataJobInputOutput.ASPECT_NAME, "add", - path=f"/outputDatasetEdges/{self.quote(str(output))}", + path=f"/outputDatasetEdges/{self.quote(output_urn)}", value=output_edge, ) return self diff --git a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json index f820efc7399492..e026664a78e0be 100644 --- a/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json +++ b/metadata-ingestion/tests/integration/nifi/nifi_mces_golden_standalone.json @@ -60,15 +60,7 @@ "op": "add", "path": "/inputDatajobEdges/urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)", "value": { - "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)", - "created": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - }, - "lastModified": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - } + "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),91d59f03-1c2b-3f3f-48bc-f89296a328bd)" } } ] @@ -178,30 +170,14 @@ "op": "add", "path": "/inputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", "value": { - "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "created": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - }, - "lastModified": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - } + "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)" } }, { "op": "add", "path": "/inputDatajobEdges/urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)", "value": { - "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)", - "created": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - }, - "lastModified": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - } + "destinationUrn": "urn:li:dataJob:(urn:li:dataFlow:(nifi,803ebb92-017d-1000-2961-4bdaa27a3ba0,PROD),cb7693ed-f93b-3340-3776-fe80e6283ddc)" } } ] @@ -287,15 +263,7 @@ "op": "add", "path": "/inputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", "value": { - "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)", - "created": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - }, - "lastModified": { - "time": 1638532800000, - "actor": "urn:li:corpuser:datahub" - } + "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,enriched-topical-chat,PROD)" } } ] diff --git a/metadata-ingestion/tests/unit/patch/test_patch_builder.py b/metadata-ingestion/tests/unit/patch/test_patch_builder.py index 267da6cdd5d205..f4bf501e0714d0 100644 --- a/metadata-ingestion/tests/unit/patch/test_patch_builder.py +++ b/metadata-ingestion/tests/unit/patch/test_patch_builder.py @@ -1,5 +1,6 @@ import json import pathlib +from typing import Any, Dict, Union import pytest from freezegun.api import freeze_time @@ -15,7 +16,9 @@ ) from datahub.ingestion.sink.file import write_metadata_file from datahub.metadata.schema_classes import ( + AuditStampClass, DatasetLineageTypeClass, + EdgeClass, FineGrainedLineageClass, FineGrainedLineageDownstreamTypeClass, FineGrainedLineageUpstreamTypeClass, @@ -182,8 +185,66 @@ def test_basic_dashboard_patch_builder(): ] +@pytest.mark.parametrize( + "created_on,last_modified,expected_actor", + [ + (1586847600000, 1586847600000, "urn:li:corpuser:datahub"), + (None, None, "urn:li:corpuser:datahub"), + (1586847600000, None, "urn:li:corpuser:datahub"), + (None, 1586847600000, "urn:li:corpuser:datahub"), + ], + ids=["both_timestamps", "no_timestamps", "only_created", "only_modified"], +) @freeze_time("2020-04-14 07:00:00") -def test_datajob_patch_builder(): +def test_datajob_patch_builder(created_on, last_modified, expected_actor): + def make_edge_or_urn(urn: str) -> Union[EdgeClass, str]: + if created_on or last_modified: + return EdgeClass( + destinationUrn=str(urn), + created=( + AuditStampClass( + time=created_on, + actor=expected_actor, + ) + if created_on + else None + ), + lastModified=( + AuditStampClass( + time=last_modified, + actor=expected_actor, + ) + if last_modified + else None + ), + ) + return urn + + def get_edge_expectation(urn: str) -> Dict[str, Any]: + if created_on or last_modified: + expected = { + "destinationUrn": str(urn), + "created": ( + AuditStampClass( + time=created_on, + actor=expected_actor, + ).to_obj() + if created_on + else None + ), + "lastModified": ( + AuditStampClass( + time=last_modified, + actor=expected_actor, + ).to_obj() + if last_modified + else None + ), + } + # filter out None values + return {k: v for k, v in expected.items() if v is not None} + return {"destinationUrn": str(urn)} + flow_urn = make_data_flow_urn( orchestrator="nifi", flow_id="252C34e5af19-0192-1000-b248-b1abee565b5d" ) @@ -193,13 +254,19 @@ def test_datajob_patch_builder(): patcher = DataJobPatchBuilder(job_urn) patcher.add_output_dataset( - "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)" + make_edge_or_urn( + "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)" + ) ) patcher.add_output_dataset( - "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)" + make_edge_or_urn( + "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)" + ) ) patcher.add_output_dataset( - "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)" + make_edge_or_urn( + "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)" + ) ) assert patcher.build() == [ @@ -214,47 +281,23 @@ def test_datajob_patch_builder(): { "op": "add", "path": "/outputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket~1folder1,DEV)", - "value": { - "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)", - "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub", - }, - "lastModified": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub", - }, - }, + "value": get_edge_expectation( + "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder1,DEV)" + ), }, { "op": "add", "path": "/outputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket~1folder3,DEV)", - "value": { - "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)", - "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub", - }, - "lastModified": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub", - }, - }, + "value": get_edge_expectation( + "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder3,DEV)" + ), }, { "op": "add", "path": "/outputDatasetEdges/urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket~1folder2,DEV)", - "value": { - "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)", - "created": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub", - }, - "lastModified": { - "time": 1586847600000, - "actor": "urn:li:corpuser:datahub", - }, - }, + "value": get_edge_expectation( + "urn:li:dataset:(urn:li:dataPlatform:s3,output-bucket/folder2,DEV)" + ), }, ] ).encode("utf-8"), diff --git a/smoke-test/tests/patch/test_datajob_patches.py b/smoke-test/tests/patch/test_datajob_patches.py index eb129e1e032125..e025c8a2aebeb5 100644 --- a/smoke-test/tests/patch/test_datajob_patches.py +++ b/smoke-test/tests/patch/test_datajob_patches.py @@ -1,5 +1,7 @@ +import time import uuid +import datahub.metadata.schema_classes as models from datahub.emitter.mce_builder import make_data_job_urn, make_dataset_urn from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.metadata.schema_classes import ( @@ -136,3 +138,95 @@ def test_datajob_inputoutput_dataset_patch(graph_client): inputoutput_lineage_read.inputDatasetEdges[0].destinationUrn == other_dataset_urn ) + + +def test_datajob_multiple_inputoutput_dataset_patch(graph_client): + """Test creating a data job with multiple input and output datasets and verifying the aspects.""" + # Create the data job + datajob_urn = "urn:li:dataJob:(urn:li:dataFlow:(airflow,training,default),training)" + + # Create input and output dataset URNs + input_datasets = ["input_data_1", "input_data_2"] + output_datasets = ["output_data_1", "output_data_2"] + + input_dataset_urns = [ + make_dataset_urn(platform="s3", name=f"test_patch_{dataset}", env="PROD") + for dataset in input_datasets + ] + output_dataset_urns = [ + make_dataset_urn(platform="s3", name=f"test_patch_{dataset}", env="PROD") + for dataset in output_datasets + ] + + # Create edges for datasets + def make_edge(urn, generate_auditstamp=False): + audit_stamp = models.AuditStampClass( + time=int(time.time() * 1000.0), + actor="urn:li:corpuser:datahub", + ) + return EdgeClass( + destinationUrn=str(urn), + lastModified=audit_stamp if generate_auditstamp else None, + ) + + # Initialize empty input/output lineage + initial_lineage = DataJobInputOutputClass( + inputDatasets=[], outputDatasets=[], inputDatasetEdges=[], outputDatasetEdges=[] + ) + + # Emit initial lineage + mcpw = MetadataChangeProposalWrapper(entityUrn=datajob_urn, aspect=initial_lineage) + graph_client.emit_mcp(mcpw) + + # Create patches for input and output datasets + patch_builder = DataJobPatchBuilder(datajob_urn) + for input_urn in input_dataset_urns: + patch_builder.add_input_dataset(make_edge(input_urn)) + for output_urn in output_dataset_urns: + patch_builder.add_output_dataset(make_edge(output_urn)) + + # Apply patches + for patch_mcp in patch_builder.build(): + graph_client.emit_mcp(patch_mcp) + + # Verify the lineage was correctly applied + lineage_aspect = graph_client.get_aspect( + entity_urn=datajob_urn, + aspect_type=DataJobInputOutputClass, + ) + + # Assert lineage was created + assert lineage_aspect is not None + assert lineage_aspect.inputDatasetEdges is not None + assert lineage_aspect.outputDatasetEdges is not None + + # Verify input datasets + assert len(lineage_aspect.inputDatasetEdges) == len(input_datasets) + input_urns = {edge.destinationUrn for edge in lineage_aspect.inputDatasetEdges} + expected_input_urns = {str(urn) for urn in input_dataset_urns} + assert input_urns == expected_input_urns + + # Verify output datasets + assert len(lineage_aspect.outputDatasetEdges) == len(output_datasets) + output_urns = {edge.destinationUrn for edge in lineage_aspect.outputDatasetEdges} + expected_output_urns = {str(urn) for urn in output_dataset_urns} + assert output_urns == expected_output_urns + + # Test updating the same datasets again (idempotency) + patch_builder = DataJobPatchBuilder(datajob_urn) + for input_urn in input_dataset_urns: + patch_builder.add_input_dataset(make_edge(input_urn)) + for output_urn in output_dataset_urns: + patch_builder.add_output_dataset(make_edge(output_urn)) + + for patch_mcp in patch_builder.build(): + graph_client.emit_mcp(patch_mcp) + + # Verify the aspect hasn't changed + updated_lineage_aspect = graph_client.get_aspect( + entity_urn=datajob_urn, + aspect_type=DataJobInputOutputClass, + ) + + assert updated_lineage_aspect is not None + assert updated_lineage_aspect.to_obj() == lineage_aspect.to_obj() From 43c8460fe528f70ccb3d9aaa8582d619de0c03e7 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 9 Dec 2024 21:34:04 -0600 Subject: [PATCH 157/174] fix(plugin-logging): adjust error logging in plugin registry (#12064) --- .../metadata/models/registry/PluginEntityRegistryLoader.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java index 4f2e5a106ae792..531537852109b0 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoader.java @@ -6,7 +6,6 @@ import com.linkedin.metadata.models.registry.config.LoadStatus; import com.linkedin.util.Pair; import java.io.File; -import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.nio.file.Files; @@ -204,8 +203,8 @@ private void loadOneRegistry( loadResultBuilder.plugins(entityRegistry.getPluginFactory().getPluginLoadResult()); log.info("Loaded registry {} successfully", entityRegistry); - } catch (RuntimeException | EntityRegistryException | IOException e) { - log.debug("{}: Failed to load registry {} with {}", this, registryName, e.getMessage()); + } catch (Exception | EntityRegistryException e) { + log.error("{}: Failed to load registry {} with {}", this, registryName, e.getMessage(), e); StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw); e.printStackTrace(pw); From 638a0e370ec595edbfe12099c560675dc9510ccf Mon Sep 17 00:00:00 2001 From: Chakru <161002324+chakru-r@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:31:35 +0530 Subject: [PATCH 158/174] build(metadata-events): fix shell interpreter mismatch in build script (#12066) --- metadata-events/mxe-schemas/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-events/mxe-schemas/build.gradle b/metadata-events/mxe-schemas/build.gradle index ab0ea8b649e9d4..6dfe69a420242f 100644 --- a/metadata-events/mxe-schemas/build.gradle +++ b/metadata-events/mxe-schemas/build.gradle @@ -25,7 +25,7 @@ task copyOriginalAvsc(type: Copy, dependsOn: generateAvroSchema) { } task renameNamespace(type: Exec, dependsOn: copyOriginalAvsc) { - commandLine 'sh', './rename-namespace.sh' + commandLine 'bash', './rename-namespace.sh' } build.dependsOn renameNamespace @@ -34,4 +34,4 @@ clean { project.delete('src/main/pegasus') project.delete('src/mainGeneratedAvroSchema/avro') project.delete('src/renamed/avro') -} \ No newline at end of file +} From 0a2ac70d38377a77eecf70c84b606f013594258f Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Mon, 9 Dec 2024 23:15:07 -0600 Subject: [PATCH 159/174] fix(entity-service): handle no-op system-metadata batches (#12055) --- .../metadata/aspect/batch/AspectsBatch.java | 19 +- .../metadata/aspect/batch/BatchItem.java | 7 + .../test/metadata/aspect/batch/TestMCL.java | 21 + .../test/metadata/aspect/batch/TestMCP.java | 38 ++ .../metadata/entity/EntityAspect.java | 25 + .../entity/ebean/batch/AspectsBatchImpl.java | 40 +- .../entity/ebean/batch/ChangeItemImpl.java | 13 +- .../entity/ebean/batch/DeleteItemImpl.java | 6 + .../entity/ebean/batch/MCLItemImpl.java | 6 + .../entity/ebean/batch/PatchItemImpl.java | 9 +- .../entity/ebean/batch/ProposedItem.java | 27 + .../ebean/batch/AspectsBatchImplTest.java | 10 +- .../metadata/entity/EntityServiceImpl.java | 197 ++++--- .../metadata/entity/TransactionContext.java | 6 + .../entity/cassandra/CassandraAspectDao.java | 4 +- .../metadata/entity/ebean/EbeanAspectDao.java | 4 +- .../metadata/AspectGenerationUtils.java | 6 +- .../entity/EbeanEntityServiceTest.java | 378 ------------- .../metadata/entity/EntityServiceTest.java | 518 +++++++++++++++++- .../ebean/batch/ChangeItemImplTest.java | 41 ++ .../SchemaFieldSideEffectTest.java | 16 +- .../src/main/resources/application.yaml | 2 +- .../metadata/utils/GenericRecordUtils.java | 19 + 23 files changed, 927 insertions(+), 485 deletions(-) create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImplTest.java diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java index 30f5dce379a077..6ce6a9a5730385 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/AspectsBatch.java @@ -28,10 +28,12 @@ public interface AspectsBatch { Collection getItems(); + Collection getInitialItems(); + RetrieverContext getRetrieverContext(); /** - * Returns MCP items. Could be patch, upsert, etc. + * Returns MCP items. Could be one of patch, upsert, etc. * * @return batch items */ @@ -160,13 +162,24 @@ static Stream applyMCLSideEffects( } default boolean containsDuplicateAspects() { - return getItems().stream() - .map(i -> String.format("%s_%s", i.getClass().getName(), i.hashCode())) + return getInitialItems().stream() + .map(i -> String.format("%s_%s", i.getClass().getSimpleName(), i.hashCode())) .distinct() .count() != getItems().size(); } + default Map> duplicateAspects() { + return getInitialItems().stream() + .collect( + Collectors.groupingBy( + i -> String.format("%s_%s", i.getClass().getSimpleName(), i.hashCode()))) + .entrySet() + .stream() + .filter(entry -> entry.getValue() != null && entry.getValue().size() > 1) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + default Map> getUrnAspectsMap() { return getItems().stream() .map(aspect -> Pair.of(aspect.getUrn().toString(), aspect.getAspectName())) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java index a6dfbc277e12ec..7f0a849a0eda1d 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/aspect/batch/BatchItem.java @@ -23,4 +23,11 @@ public interface BatchItem extends ReadItem { */ @Nonnull ChangeType getChangeType(); + + /** + * Determines if this item is a duplicate of another item in terms of the operation it represents + * to the database.Each implementation can define what constitutes a duplicate based on its + * specific fields which are persisted. + */ + boolean isDatabaseDuplicateOf(BatchItem other); } diff --git a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java index 7dd889c48b8747..6643a9de58562b 100644 --- a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java +++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCL.java @@ -4,10 +4,12 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.MCLItem; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.mxe.MetadataChangeLog; +import java.util.Objects; import javax.annotation.Nonnull; import lombok.Builder; import lombok.Getter; @@ -29,4 +31,23 @@ public class TestMCL implements MCLItem { public String getAspectName() { return getAspectSpec().getName(); } + + @Override + public boolean isDatabaseDuplicateOf(BatchItem other) { + return equals(other); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TestMCL testMCL = (TestMCL) o; + return Objects.equals(metadataChangeLog, testMCL.metadataChangeLog); + } + + @Override + public int hashCode() { + return Objects.hashCode(metadataChangeLog); + } } diff --git a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java index e562390a959a74..5b714bdbf0b478 100644 --- a/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java +++ b/entity-registry/src/testFixtures/java/com/linkedin/test/metadata/aspect/batch/TestMCP.java @@ -6,6 +6,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.ReadItem; @@ -21,6 +22,7 @@ import java.net.URISyntaxException; import java.util.Collection; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -140,4 +142,40 @@ public Map getHeaders() { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) .orElse(headers); } + + @Override + public boolean isDatabaseDuplicateOf(BatchItem other) { + return equals(other); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TestMCP testMCP = (TestMCP) o; + return urn.equals(testMCP.urn) + && DataTemplateUtil.areEqual(recordTemplate, testMCP.recordTemplate) + && Objects.equals(systemAspect, testMCP.systemAspect) + && Objects.equals(previousSystemAspect, testMCP.previousSystemAspect) + && Objects.equals(auditStamp, testMCP.auditStamp) + && Objects.equals(changeType, testMCP.changeType) + && Objects.equals(metadataChangeProposal, testMCP.metadataChangeProposal); + } + + @Override + public int hashCode() { + int result = urn.hashCode(); + result = 31 * result + Objects.hashCode(recordTemplate); + result = 31 * result + Objects.hashCode(systemAspect); + result = 31 * result + Objects.hashCode(previousSystemAspect); + result = 31 * result + Objects.hashCode(auditStamp); + result = 31 * result + Objects.hashCode(changeType); + result = 31 * result + Objects.hashCode(metadataChangeProposal); + return result; + } } diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java index 976db4133c0043..2b67d5e92f833c 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/EntityAspect.java @@ -52,6 +52,26 @@ public class EntityAspect { private String createdFor; + @Override + public String toString() { + return "EntityAspect{" + + "urn='" + + urn + + '\'' + + ", aspect='" + + aspect + + '\'' + + ", version=" + + version + + ", metadata='" + + metadata + + '\'' + + ", systemMetadata='" + + systemMetadata + + '\'' + + '}'; + } + /** * Provide a typed EntityAspect without breaking the existing public contract with generic types. */ @@ -144,6 +164,11 @@ public EnvelopedAspect toEnvelopedAspects() { return envelopedAspect; } + @Override + public String toString() { + return entityAspect.toString(); + } + public static class EntitySystemAspectBuilder { private EntityAspect.EntitySystemAspect build() { diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java index c0d65640df2378..1af9fc1565a456 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImpl.java @@ -1,6 +1,7 @@ package com.linkedin.metadata.entity.ebean.batch; import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; @@ -15,7 +16,9 @@ import com.linkedin.metadata.models.EntitySpec; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.util.Pair; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -29,12 +32,23 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Getter @Builder(toBuilder = true) public class AspectsBatchImpl implements AspectsBatch { @Nonnull private final Collection items; - @Nonnull private final RetrieverContext retrieverContext; + @Nonnull private final Collection nonRepeatedItems; + @Getter @Nonnull private final RetrieverContext retrieverContext; + + @Override + @Nonnull + public Collection getItems() { + return nonRepeatedItems; + } + + @Override + public Collection getInitialItems() { + return items; + } /** * Convert patches to upserts, apply hooks at the aspect and batch level. @@ -207,14 +221,32 @@ public AspectsBatchImplBuilder mcps( return this; } + private static List filterRepeats(Collection items) { + List result = new ArrayList<>(); + Map, T> last = new HashMap<>(); + + for (T item : items) { + Pair urnAspect = Pair.of(item.getUrn(), item.getAspectName()); + // Check if this item is a duplicate of the previous + if (!last.containsKey(urnAspect) || !item.isDatabaseDuplicateOf(last.get(urnAspect))) { + result.add(item); + } + last.put(urnAspect, item); + } + + return result; + } + public AspectsBatchImpl build() { + this.nonRepeatedItems = filterRepeats(this.items); + ValidationExceptionCollection exceptions = - AspectsBatch.validateProposed(this.items, this.retrieverContext); + AspectsBatch.validateProposed(this.nonRepeatedItems, this.retrieverContext); if (!exceptions.isEmpty()) { throw new IllegalArgumentException("Failed to validate MCP due to: " + exceptions); } - return new AspectsBatchImpl(this.items, this.retrieverContext); + return new AspectsBatchImpl(this.items, this.nonRepeatedItems, this.retrieverContext); } } diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java index 6f45a36d1daf46..64263859e4aadb 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImpl.java @@ -3,11 +3,13 @@ import com.datahub.util.exception.ModelConversionException; import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringMap; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.SystemAspect; +import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.ChangeMCP; import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.patch.template.common.GenericPatchTemplate; @@ -269,6 +271,11 @@ private static RecordTemplate convertToRecordTemplate( } } + @Override + public boolean isDatabaseDuplicateOf(BatchItem other) { + return equals(other); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -280,13 +287,15 @@ public boolean equals(Object o) { ChangeItemImpl that = (ChangeItemImpl) o; return urn.equals(that.urn) && aspectName.equals(that.aspectName) + && changeType.equals(that.changeType) && Objects.equals(systemMetadata, that.systemMetadata) - && recordTemplate.equals(that.recordTemplate); + && Objects.equals(auditStamp, that.auditStamp) + && DataTemplateUtil.areEqual(recordTemplate, that.recordTemplate); } @Override public int hashCode() { - return Objects.hash(urn, aspectName, systemMetadata, recordTemplate); + return Objects.hash(urn, aspectName, changeType, systemMetadata, auditStamp, recordTemplate); } @Override diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java index 9c1ded284fa0bd..40bcb0fa8ed2d1 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/DeleteItemImpl.java @@ -6,6 +6,7 @@ import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.SystemAspect; +import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.ChangeMCP; import com.linkedin.metadata.entity.EntityApiUtils; import com.linkedin.metadata.entity.EntityAspect; @@ -115,6 +116,11 @@ public DeleteItemImpl build(AspectRetriever aspectRetriever) { } } + @Override + public boolean isDatabaseDuplicateOf(BatchItem other) { + return equals(other); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java index a5afd4651ed2c4..85923a28a64be5 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/MCLItemImpl.java @@ -5,6 +5,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.MCLItem; import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.entity.AspectUtils; @@ -158,6 +159,11 @@ private static Pair convertToRecordTemplate( } } + @Override + public boolean isDatabaseDuplicateOf(BatchItem other) { + return equals(other); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java index ec0a8422e3c4a2..2543d99ac6af37 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/PatchItemImpl.java @@ -14,6 +14,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; +import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.aspect.batch.PatchMCP; import com.linkedin.metadata.aspect.patch.template.AspectTemplateEngine; @@ -216,6 +217,11 @@ public static JsonPatch convertToJsonPatch(MetadataChangeProposal mcp) { } } + @Override + public boolean isDatabaseDuplicateOf(BatchItem other) { + return equals(other); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -228,12 +234,13 @@ public boolean equals(Object o) { return urn.equals(that.urn) && aspectName.equals(that.aspectName) && Objects.equals(systemMetadata, that.systemMetadata) + && auditStamp.equals(that.auditStamp) && patch.equals(that.patch); } @Override public int hashCode() { - return Objects.hash(urn, aspectName, systemMetadata, patch); + return Objects.hash(urn, aspectName, systemMetadata, auditStamp, patch); } @Override diff --git a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java index 88187ef159f233..370f1f6f073e65 100644 --- a/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java +++ b/metadata-io/metadata-io-api/src/main/java/com/linkedin/metadata/entity/ebean/batch/ProposedItem.java @@ -4,6 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.aspect.batch.BatchItem; import com.linkedin.metadata.aspect.batch.MCPItem; import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.EntitySpec; @@ -86,6 +87,32 @@ public ChangeType getChangeType() { return metadataChangeProposal.getChangeType(); } + @Override + public boolean isDatabaseDuplicateOf(BatchItem other) { + return equals(other); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ProposedItem that = (ProposedItem) o; + return metadataChangeProposal.equals(that.metadataChangeProposal) + && auditStamp.equals(that.auditStamp); + } + + @Override + public int hashCode() { + int result = metadataChangeProposal.hashCode(); + result = 31 * result + auditStamp.hashCode(); + return result; + } + public static class ProposedItemBuilder { public ProposedItem build() { // Ensure systemMetadata diff --git a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java index 96f535f2295aa4..9f57d36f800de3 100644 --- a/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java +++ b/metadata-io/metadata-io-api/src/test/java/com/linkedin/metadata/entity/ebean/batch/AspectsBatchImplTest.java @@ -6,6 +6,7 @@ import static org.testng.Assert.assertEquals; import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; import com.linkedin.common.FabricType; import com.linkedin.common.Status; import com.linkedin.common.urn.DataPlatformUrn; @@ -220,6 +221,7 @@ public void toUpsertBatchItemsPatchItemTest() { @Test public void toUpsertBatchItemsProposedItemTest() { + AuditStamp auditStamp = AuditStampUtils.createDefaultAuditStamp(); List testItems = List.of( ProposedItem.builder() @@ -239,7 +241,7 @@ public void toUpsertBatchItemsProposedItemTest() { ByteString.copyString( "{\"foo\":\"bar\"}", StandardCharsets.UTF_8))) .setSystemMetadata(new SystemMetadata())) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .auditStamp(auditStamp) .build(), ProposedItem.builder() .entitySpec(testRegistry.getEntitySpec(DATASET_ENTITY_NAME)) @@ -258,7 +260,7 @@ public void toUpsertBatchItemsProposedItemTest() { ByteString.copyString( "{\"foo\":\"bar\"}", StandardCharsets.UTF_8))) .setSystemMetadata(new SystemMetadata())) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .auditStamp(auditStamp) .build()); AspectsBatchImpl testBatch = @@ -280,7 +282,7 @@ public void toUpsertBatchItemsProposedItemTest() { testRegistry .getEntitySpec(DATASET_ENTITY_NAME) .getAspectSpec(STATUS_ASPECT_NAME)) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .auditStamp(auditStamp) .systemMetadata(testItems.get(0).getSystemMetadata().setVersion("1")) .recordTemplate(new Status().setRemoved(false)) .build(mockAspectRetriever), @@ -295,7 +297,7 @@ public void toUpsertBatchItemsProposedItemTest() { testRegistry .getEntitySpec(DATASET_ENTITY_NAME) .getAspectSpec(STATUS_ASPECT_NAME)) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .auditStamp(auditStamp) .systemMetadata(testItems.get(1).getSystemMetadata().setVersion("1")) .recordTemplate(new Status().setRemoved(false)) .build(mockAspectRetriever))), diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index bf3481205fb5ab..059a6b7ed0aea3 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -854,7 +854,7 @@ private List ingestAspectsToLocalDB( boolean overwrite) { if (inputBatch.containsDuplicateAspects()) { - log.warn(String.format("Batch contains duplicates: %s", inputBatch)); + log.warn("Batch contains duplicates: {}", inputBatch.duplicateAspects()); MetricUtils.counter(EntityServiceImpl.class, "batch_with_duplicate").inc(); } @@ -968,39 +968,20 @@ private List ingestAspectsToLocalDB( writeItem.getAspectSpec(), databaseAspect); - final UpdateAspectResult result; /* This condition is specifically for an older conditional write ingestAspectIfNotPresent() overwrite is always true otherwise */ if (overwrite || databaseAspect == null) { - result = - Optional.ofNullable( - ingestAspectToLocalDB( - txContext, writeItem, databaseSystemAspect)) - .map( - optResult -> - optResult.toBuilder().request(writeItem).build()) - .orElse(null); - - } else { - RecordTemplate oldValue = databaseSystemAspect.getRecordTemplate(); - SystemMetadata oldMetadata = databaseSystemAspect.getSystemMetadata(); - result = - UpdateAspectResult.builder() - .urn(writeItem.getUrn()) - .request(writeItem) - .oldValue(oldValue) - .newValue(oldValue) - .oldSystemMetadata(oldMetadata) - .newSystemMetadata(oldMetadata) - .operation(MetadataAuditOperation.UPDATE) - .auditStamp(writeItem.getAuditStamp()) - .maxVersion(databaseAspect.getVersion()) - .build(); + return Optional.ofNullable( + ingestAspectToLocalDB( + txContext, writeItem, databaseSystemAspect)) + .map( + optResult -> optResult.toBuilder().request(writeItem).build()) + .orElse(null); } - return result; + return null; }) .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -1051,7 +1032,8 @@ This condition is specifically for an older conditional write ingestAspectIfNotP } } else { MetricUtils.counter(EntityServiceImpl.class, "batch_empty_transaction").inc(); - log.warn("Empty transaction detected. {}", inputBatch); + // This includes no-op batches. i.e. patch removing non-existent items + log.debug("Empty transaction detected"); } return upsertResults; @@ -1150,7 +1132,7 @@ public RecordTemplate ingestAspectIfNotPresent( .build(); List ingested = ingestAspects(opContext, aspectsBatch, true, false); - return ingested.stream().findFirst().get().getNewValue(); + return ingested.stream().findFirst().map(UpdateAspectResult::getNewValue).orElse(null); } /** @@ -2525,6 +2507,14 @@ private UpdateAspectResult ingestAspectToLocalDB( @Nonnull final ChangeMCP writeItem, @Nullable final EntityAspect.EntitySystemAspect databaseAspect) { + if (writeItem.getRecordTemplate() == null) { + log.error( + "Unexpected write of null aspect with name {}, urn {}", + writeItem.getAspectName(), + writeItem.getUrn()); + return null; + } + // Set the "last run id" to be the run id provided with the new system metadata. This will be // stored in index // for all aspects that have a run id, regardless of whether they change. @@ -2533,9 +2523,6 @@ private UpdateAspectResult ingestAspectToLocalDB( .setLastRunId(writeItem.getSystemMetadata().getRunId(GetMode.NULL), SetMode.IGNORE_NULL); // 2. Compare the latest existing and new. - final RecordTemplate databaseValue = - databaseAspect == null ? null : databaseAspect.getRecordTemplate(); - final EntityAspect.EntitySystemAspect previousBatchAspect = (EntityAspect.EntitySystemAspect) writeItem.getPreviousSystemAspect(); final RecordTemplate previousValue = @@ -2544,45 +2531,86 @@ private UpdateAspectResult ingestAspectToLocalDB( // 3. If there is no difference between existing and new, we just update // the lastObserved in system metadata. RunId should stay as the original runId if (previousValue != null - && DataTemplateUtil.areEqual(databaseValue, writeItem.getRecordTemplate())) { + && DataTemplateUtil.areEqual(previousValue, writeItem.getRecordTemplate())) { - SystemMetadata latestSystemMetadata = previousBatchAspect.getSystemMetadata(); - latestSystemMetadata.setLastObserved(writeItem.getSystemMetadata().getLastObserved()); - latestSystemMetadata.setLastRunId( - writeItem.getSystemMetadata().getLastRunId(GetMode.NULL), SetMode.IGNORE_NULL); - - previousBatchAspect - .getEntityAspect() - .setSystemMetadata(RecordUtils.toJsonString(latestSystemMetadata)); - - log.info( - "Ingesting aspect with name {}, urn {}", - previousBatchAspect.getAspectName(), - previousBatchAspect.getUrn()); - aspectDao.saveAspect(txContext, previousBatchAspect.getEntityAspect(), false); - - // metrics - aspectDao.incrementWriteMetrics( - previousBatchAspect.getAspectName(), - 1, - previousBatchAspect.getMetadataRaw().getBytes(StandardCharsets.UTF_8).length); + Optional latestSystemMetadataDiff = + systemMetadataDiff( + txContext, + previousBatchAspect.getSystemMetadata(), + writeItem.getSystemMetadata(), + databaseAspect == null ? null : databaseAspect.getSystemMetadata()); + + if (latestSystemMetadataDiff.isPresent()) { + // Update previous version since that is what is re-written + previousBatchAspect + .getEntityAspect() + .setSystemMetadata(RecordUtils.toJsonString(latestSystemMetadataDiff.get())); + + // Inserts & update order is not guaranteed, flush the insert for potential updates within + // same tx + if (databaseAspect == null && txContext != null) { + conditionalLogLevel( + txContext, + String.format( + "Flushing for systemMetadata update aspect with name %s, urn %s", + writeItem.getAspectName(), writeItem.getUrn())); + txContext.flush(); + } - return UpdateAspectResult.builder() - .urn(writeItem.getUrn()) - .oldValue(previousValue) - .newValue(previousValue) - .oldSystemMetadata(previousBatchAspect.getSystemMetadata()) - .newSystemMetadata(latestSystemMetadata) - .operation(MetadataAuditOperation.UPDATE) - .auditStamp(writeItem.getAuditStamp()) - .maxVersion(0) - .build(); + conditionalLogLevel( + txContext, + String.format( + "Update aspect with name %s, urn %s, txContext: %s, databaseAspect: %s, newAspect: %s", + previousBatchAspect.getAspectName(), + previousBatchAspect.getUrn(), + txContext != null, + databaseAspect == null ? null : databaseAspect.getEntityAspect(), + previousBatchAspect.getEntityAspect())); + aspectDao.saveAspect(txContext, previousBatchAspect.getEntityAspect(), false); + + // metrics + aspectDao.incrementWriteMetrics( + previousBatchAspect.getAspectName(), + 1, + previousBatchAspect.getMetadataRaw().getBytes(StandardCharsets.UTF_8).length); + + return UpdateAspectResult.builder() + .urn(writeItem.getUrn()) + .oldValue(previousValue) + .newValue(previousValue) + .oldSystemMetadata(previousBatchAspect.getSystemMetadata()) + .newSystemMetadata(latestSystemMetadataDiff.get()) + .operation(MetadataAuditOperation.UPDATE) + .auditStamp(writeItem.getAuditStamp()) + .maxVersion(0) + .build(); + } else { + MetricUtils.counter(EntityServiceImpl.class, "batch_with_noop_sysmetadata").inc(); + return null; + } } // 4. Save the newValue as the latest version - if (!DataTemplateUtil.areEqual(databaseValue, writeItem.getRecordTemplate())) { - log.debug( - "Ingesting aspect with name {}, urn {}", writeItem.getAspectName(), writeItem.getUrn()); + if (writeItem.getRecordTemplate() != null + && !DataTemplateUtil.areEqual(previousValue, writeItem.getRecordTemplate())) { + conditionalLogLevel( + txContext, + String.format( + "Insert aspect with name %s, urn %s", writeItem.getAspectName(), writeItem.getUrn())); + + // Inserts & update order is not guaranteed, flush the insert for potential updates within + // same tx + if (databaseAspect == null + && !ASPECT_LATEST_VERSION.equals(writeItem.getNextAspectVersion()) + && txContext != null) { + conditionalLogLevel( + txContext, + String.format( + "Flushing for update aspect with name %s, urn %s", + writeItem.getAspectName(), writeItem.getUrn())); + txContext.flush(); + } + String newValueStr = EntityApiUtils.toJsonAspect(writeItem.getRecordTemplate()); long versionOfOld = aspectDao.saveLatestAspect( @@ -2630,4 +2658,41 @@ private static boolean shouldAspectEmitChangeLog(@Nonnull final AspectSpec aspec aspectSpec.getRelationshipFieldSpecs(); return relationshipFieldSpecs.stream().anyMatch(RelationshipFieldSpec::isLineageRelationship); } + + private static Optional systemMetadataDiff( + @Nullable TransactionContext txContext, + @Nullable SystemMetadata previous, + @Nonnull SystemMetadata current, + @Nullable SystemMetadata database) { + + SystemMetadata latestSystemMetadata = GenericRecordUtils.copy(previous, SystemMetadata.class); + + latestSystemMetadata.setLastRunId(previous.getRunId(), SetMode.REMOVE_IF_NULL); + latestSystemMetadata.setLastObserved(current.getLastObserved(), SetMode.IGNORE_NULL); + latestSystemMetadata.setRunId(current.getRunId(), SetMode.REMOVE_IF_NULL); + + if (!DataTemplateUtil.areEqual(latestSystemMetadata, previous) + && !DataTemplateUtil.areEqual(latestSystemMetadata, database)) { + + conditionalLogLevel( + txContext, + String.format( + "systemMetdataDiff: %s != %s AND %s", + RecordUtils.toJsonString(latestSystemMetadata), + previous == null ? null : RecordUtils.toJsonString(previous), + database == null ? null : RecordUtils.toJsonString(database))); + + return Optional.of(latestSystemMetadata); + } + + return Optional.empty(); + } + + private static void conditionalLogLevel(@Nullable TransactionContext txContext, String message) { + if (txContext != null && txContext.getFailedAttempts() > 1) { + log.warn(message); + } else { + log.debug(message); + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java index 69f2f1c8981c03..6897c9152e9a25 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/TransactionContext.java @@ -66,4 +66,10 @@ public void commitAndContinue() { } success(); } + + public void flush() { + if (tx != null) { + tx.flush(); + } + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java index 9e7387947a9547..a00482acda62e2 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/cassandra/CassandraAspectDao.java @@ -590,7 +590,7 @@ public long saveLatestAspect( // Save oldValue as the largest version + 1 long largestVersion = ASPECT_LATEST_VERSION; BatchStatement batch = BatchStatement.newInstance(BatchType.UNLOGGED); - if (oldAspectMetadata != null && oldTime != null) { + if (!ASPECT_LATEST_VERSION.equals(nextVersion) && oldTime != null) { largestVersion = nextVersion; final EntityAspect aspect = new EntityAspect( @@ -616,7 +616,7 @@ public long saveLatestAspect( newTime, newActor, newImpersonator); - batch = batch.add(generateSaveStatement(aspect, oldAspectMetadata == null)); + batch = batch.add(generateSaveStatement(aspect, ASPECT_LATEST_VERSION.equals(nextVersion))); _cqlSession.execute(batch); return largestVersion; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java index 6233bf5e0e35cf..729d0e61cb2c00 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/ebean/EbeanAspectDao.java @@ -165,7 +165,7 @@ public long saveLatestAspect( } // Save oldValue as the largest version + 1 long largestVersion = ASPECT_LATEST_VERSION; - if (oldAspectMetadata != null && oldTime != null) { + if (!ASPECT_LATEST_VERSION.equals(nextVersion) && oldTime != null) { largestVersion = nextVersion; saveAspect( txContext, @@ -191,7 +191,7 @@ public long saveLatestAspect( newTime, newSystemMetadata, ASPECT_LATEST_VERSION, - oldAspectMetadata == null); + ASPECT_LATEST_VERSION.equals(nextVersion)); return largestVersion; } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java index 346a1eef845923..395c040f288111 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/AspectGenerationUtils.java @@ -34,19 +34,19 @@ public static SystemMetadata createSystemMetadata() { } @Nonnull - public static SystemMetadata createSystemMetadata(long nextAspectVersion) { + public static SystemMetadata createSystemMetadata(int nextAspectVersion) { return createSystemMetadata( 1625792689, "run-123", "run-123", String.valueOf(nextAspectVersion)); } @Nonnull - public static SystemMetadata createSystemMetadata(long lastObserved, @Nonnull String runId) { + public static SystemMetadata createSystemMetadata(int lastObserved, @Nonnull String runId) { return createSystemMetadata(lastObserved, runId, runId, null); } @Nonnull public static SystemMetadata createSystemMetadata( - long lastObserved, + int lastObserved, // for test comparison must be int @Nonnull String runId, @Nonnull String lastRunId, @Nullable String version) { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java index a1000fd02abfe1..aa42545fa0e46f 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EbeanEntityServiceTest.java @@ -1,10 +1,6 @@ package com.linkedin.metadata.entity; -import static com.linkedin.metadata.Constants.APP_SOURCE; import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME; -import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME; -import static com.linkedin.metadata.Constants.GLOBAL_TAGS_ASPECT_NAME; -import static com.linkedin.metadata.Constants.METADATA_TESTS_SOURCE; import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; import static org.mockito.Mockito.mock; import static org.testng.Assert.assertEquals; @@ -12,36 +8,27 @@ import static org.testng.Assert.assertTrue; import com.linkedin.common.AuditStamp; -import com.linkedin.common.GlobalTags; import com.linkedin.common.Status; -import com.linkedin.common.TagAssociation; -import com.linkedin.common.TagAssociationArray; -import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.StringMap; import com.linkedin.entity.EnvelopedAspect; import com.linkedin.identity.CorpUserInfo; import com.linkedin.metadata.AspectGenerationUtils; import com.linkedin.metadata.Constants; import com.linkedin.metadata.EbeanTestUtils; -import com.linkedin.metadata.aspect.patch.GenericJsonPatch; -import com.linkedin.metadata.aspect.patch.PatchOperationType; import com.linkedin.metadata.config.EbeanConfiguration; import com.linkedin.metadata.config.PreProcessHooks; import com.linkedin.metadata.entity.ebean.EbeanAspectDao; import com.linkedin.metadata.entity.ebean.EbeanRetentionService; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; -import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl; import com.linkedin.metadata.event.EventProducer; import com.linkedin.metadata.key.CorpUserKey; import com.linkedin.metadata.models.registry.EntityRegistryException; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.service.UpdateIndicesService; -import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.PegasusUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.mxe.SystemMetadata; @@ -64,7 +51,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Triple; import org.testng.Assert; import org.testng.annotations.BeforeMethod; @@ -396,360 +382,6 @@ public void testSystemMetadataDuplicateKey() throws Exception { "Expected version 0 with systemMeta version 3 accounting for the the collision"); } - @Test - public void testBatchDuplicate() throws Exception { - Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:batchDuplicateTest"); - SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); - ChangeItemImpl item1 = - ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(STATUS_ASPECT_NAME) - .recordTemplate(new Status().setRemoved(true)) - .systemMetadata(systemMetadata.copy()) - .auditStamp(TEST_AUDIT_STAMP) - .build(TestOperationContexts.emptyAspectRetriever(null)); - ChangeItemImpl item2 = - ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(STATUS_ASPECT_NAME) - .recordTemplate(new Status().setRemoved(false)) - .systemMetadata(systemMetadata.copy()) - .auditStamp(TEST_AUDIT_STAMP) - .build(TestOperationContexts.emptyAspectRetriever(null)); - _entityServiceImpl.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(item1, item2)) - .build(), - false, - true); - - // List aspects urns - ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 2); - - assertEquals(batch.getStart().intValue(), 0); - assertEquals(batch.getCount().intValue(), 1); - assertEquals(batch.getTotal().intValue(), 1); - assertEquals(batch.getEntities().size(), 1); - assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); - - EnvelopedAspect envelopedAspect = - _entityServiceImpl.getLatestEnvelopedAspect( - opContext, CORP_USER_ENTITY_NAME, entityUrn, STATUS_ASPECT_NAME); - assertEquals( - envelopedAspect.getSystemMetadata().getVersion(), - "2", - "Expected version 2 accounting for duplicates"); - assertEquals( - envelopedAspect.getValue().toString(), - "{removed=false}", - "Expected 2nd item to be the latest"); - } - - @Test - public void testBatchPatchWithTrailingNoOp() throws Exception { - Urn entityUrn = - UrnUtils.getUrn( - "urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchWithTrailingNoOp,PROD)"); - TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); - Urn tag2 = UrnUtils.getUrn("urn:li:tag:tag2"); - Urn tagOther = UrnUtils.getUrn("urn:li:tag:other"); - - SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); - - ChangeItemImpl initialAspectTag1 = - ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .recordTemplate( - new GlobalTags() - .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1)))) - .systemMetadata(systemMetadata.copy()) - .auditStamp(TEST_AUDIT_STAMP) - .build(TestOperationContexts.emptyAspectRetriever(null)); - - PatchItemImpl patchAdd2 = - PatchItemImpl.builder() - .urn(entityUrn) - .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .aspectSpec( - _testEntityRegistry - .getEntitySpec(DATASET_ENTITY_NAME) - .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) - .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) - .build() - .getJsonPatch()) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .build(_testEntityRegistry); - - PatchItemImpl patchRemoveNonExistent = - PatchItemImpl.builder() - .urn(entityUrn) - .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .aspectSpec( - _testEntityRegistry - .getEntitySpec(DATASET_ENTITY_NAME) - .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) - .patch(List.of(tagPatchOp(PatchOperationType.REMOVE, tagOther))) - .build() - .getJsonPatch()) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .build(_testEntityRegistry); - - // establish base entity - _entityServiceImpl.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(initialAspectTag1)) - .build(), - false, - true); - - _entityServiceImpl.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(patchAdd2, patchRemoveNonExistent)) - .build(), - false, - true); - - // List aspects urns - ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); - - assertEquals(batch.getStart().intValue(), 0); - assertEquals(batch.getCount().intValue(), 1); - assertEquals(batch.getTotal().intValue(), 1); - assertEquals(batch.getEntities().size(), 1); - assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); - - EnvelopedAspect envelopedAspect = - _entityServiceImpl.getLatestEnvelopedAspect( - opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); - assertEquals( - envelopedAspect.getSystemMetadata().getVersion(), - "3", - "Expected version 3. 1 - Initial, + 1 add, 1 remove"); - assertEquals( - new GlobalTags(envelopedAspect.getValue().data()) - .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), - Set.of(tag1, tag2), - "Expected both tags"); - } - - @Test - public void testBatchPatchAdd() throws Exception { - Urn entityUrn = - UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)"); - TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); - TagUrn tag2 = TagUrn.createFromString("urn:li:tag:tag2"); - TagUrn tag3 = TagUrn.createFromString("urn:li:tag:tag3"); - - SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); - - ChangeItemImpl initialAspectTag1 = - ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .recordTemplate( - new GlobalTags() - .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1)))) - .systemMetadata(systemMetadata.copy()) - .auditStamp(TEST_AUDIT_STAMP) - .build(TestOperationContexts.emptyAspectRetriever(null)); - - PatchItemImpl patchAdd3 = - PatchItemImpl.builder() - .urn(entityUrn) - .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .aspectSpec( - _testEntityRegistry - .getEntitySpec(DATASET_ENTITY_NAME) - .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) - .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag3))) - .build() - .getJsonPatch()) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .build(_testEntityRegistry); - - PatchItemImpl patchAdd2 = - PatchItemImpl.builder() - .urn(entityUrn) - .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .aspectSpec( - _testEntityRegistry - .getEntitySpec(DATASET_ENTITY_NAME) - .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) - .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) - .build() - .getJsonPatch()) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .build(_testEntityRegistry); - - PatchItemImpl patchAdd1 = - PatchItemImpl.builder() - .urn(entityUrn) - .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .aspectSpec( - _testEntityRegistry - .getEntitySpec(DATASET_ENTITY_NAME) - .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) - .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag1))) - .build() - .getJsonPatch()) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .build(_testEntityRegistry); - - // establish base entity - _entityServiceImpl.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(initialAspectTag1)) - .build(), - false, - true); - - _entityServiceImpl.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(patchAdd3, patchAdd2, patchAdd1)) - .build(), - false, - true); - - // List aspects urns - ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); - - assertEquals(batch.getStart().intValue(), 0); - assertEquals(batch.getCount().intValue(), 1); - assertEquals(batch.getTotal().intValue(), 1); - assertEquals(batch.getEntities().size(), 1); - assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); - - EnvelopedAspect envelopedAspect = - _entityServiceImpl.getLatestEnvelopedAspect( - opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); - assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "4", "Expected version 4"); - assertEquals( - new GlobalTags(envelopedAspect.getValue().data()) - .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), - Set.of(tag1, tag2, tag3), - "Expected all tags"); - } - - @Test - public void testBatchPatchAddDuplicate() throws Exception { - Urn entityUrn = - UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)"); - List initialTags = - List.of( - TagUrn.createFromString("urn:li:tag:__default_large_table"), - TagUrn.createFromString("urn:li:tag:__default_low_queries"), - TagUrn.createFromString("urn:li:tag:__default_low_changes"), - TagUrn.createFromString("urn:li:tag:!10TB+ tables")) - .stream() - .map(tag -> new TagAssociation().setTag(tag)) - .collect(Collectors.toList()); - TagUrn tag2 = TagUrn.createFromString("urn:li:tag:$ 1TB+"); - - SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); - - SystemMetadata patchSystemMetadata = new SystemMetadata(); - patchSystemMetadata.setLastObserved(systemMetadata.getLastObserved() + 1); - patchSystemMetadata.setProperties(new StringMap(Map.of(APP_SOURCE, METADATA_TESTS_SOURCE))); - - ChangeItemImpl initialAspectTag1 = - ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .recordTemplate(new GlobalTags().setTags(new TagAssociationArray(initialTags))) - .systemMetadata(systemMetadata.copy()) - .auditStamp(TEST_AUDIT_STAMP) - .build(TestOperationContexts.emptyAspectRetriever(null)); - - PatchItemImpl patchAdd2 = - PatchItemImpl.builder() - .urn(entityUrn) - .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) - .aspectName(GLOBAL_TAGS_ASPECT_NAME) - .aspectSpec( - _testEntityRegistry - .getEntitySpec(DATASET_ENTITY_NAME) - .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) - .patch( - GenericJsonPatch.builder() - .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) - .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) - .build() - .getJsonPatch()) - .systemMetadata(patchSystemMetadata) - .auditStamp(AuditStampUtils.createDefaultAuditStamp()) - .build(_testEntityRegistry); - - // establish base entity - _entityServiceImpl.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(initialAspectTag1)) - .build(), - false, - true); - - _entityServiceImpl.ingestAspects( - opContext, - AspectsBatchImpl.builder() - .retrieverContext(opContext.getRetrieverContext().get()) - .items(List.of(patchAdd2, patchAdd2)) // duplicate - .build(), - false, - true); - - // List aspects urns - ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); - - assertEquals(batch.getStart().intValue(), 0); - assertEquals(batch.getCount().intValue(), 1); - assertEquals(batch.getTotal().intValue(), 1); - assertEquals(batch.getEntities().size(), 1); - assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); - - EnvelopedAspect envelopedAspect = - _entityServiceImpl.getLatestEnvelopedAspect( - opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); - assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "3", "Expected version 3"); - assertEquals( - new GlobalTags(envelopedAspect.getValue().data()) - .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), - Stream.concat(initialTags.stream().map(TagAssociation::getTag), Stream.of(tag2)) - .collect(Collectors.toSet()), - "Expected all tags"); - } - @Test public void dataGeneratorThreadingTest() { DataGenerator dataGenerator = new DataGenerator(opContext, _entityServiceImpl); @@ -976,14 +608,4 @@ public void run() { } } } - - private static GenericJsonPatch.PatchOp tagPatchOp(PatchOperationType op, Urn tagUrn) { - GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp(); - patchOp.setOp(op.getValue()); - patchOp.setPath(String.format("/tags/%s", tagUrn)); - if (PatchOperationType.ADD.equals(op)) { - patchOp.setValue(Map.of("tag", tagUrn.toString())); - } - return patchOp; - } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index 654c448fdec946..18d277cacbbe26 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -11,14 +11,18 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; import com.linkedin.common.Owner; import com.linkedin.common.OwnerArray; import com.linkedin.common.Ownership; import com.linkedin.common.OwnershipType; import com.linkedin.common.Status; +import com.linkedin.common.TagAssociation; +import com.linkedin.common.TagAssociationArray; import com.linkedin.common.UrnArray; import com.linkedin.common.VersionedUrn; import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.TupleKey; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; @@ -42,8 +46,11 @@ import com.linkedin.metadata.aspect.CorpUserAspect; import com.linkedin.metadata.aspect.CorpUserAspectArray; import com.linkedin.metadata.aspect.VersionedAspect; +import com.linkedin.metadata.aspect.patch.GenericJsonPatch; +import com.linkedin.metadata.aspect.patch.PatchOperationType; import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; +import com.linkedin.metadata.entity.ebean.batch.PatchItemImpl; import com.linkedin.metadata.entity.restoreindices.RestoreIndicesArgs; import com.linkedin.metadata.entity.validation.ValidationApiUtils; import com.linkedin.metadata.entity.validation.ValidationException; @@ -52,10 +59,12 @@ import com.linkedin.metadata.models.AspectSpec; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistryException; +import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.run.AspectRowSummary; import com.linkedin.metadata.service.UpdateIndicesService; import com.linkedin.metadata.snapshot.CorpUserSnapshot; import com.linkedin.metadata.snapshot.Snapshot; +import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.GenericAspect; @@ -605,6 +614,9 @@ public void testReingestLineageAspect() throws Exception { entityUrn, _testEntityRegistry.getEntitySpec(entityUrn.getEntityType()).getKeyAspectSpec()))); + SystemMetadata futureSystemMetadata = AspectGenerationUtils.createSystemMetadata(1); + futureSystemMetadata.setLastObserved(futureSystemMetadata.getLastObserved() + 1); + final MetadataChangeLog restateChangeLog = new MetadataChangeLog(); restateChangeLog.setEntityType(entityUrn.getEntityType()); restateChangeLog.setEntityUrn(entityUrn); @@ -612,10 +624,10 @@ public void testReingestLineageAspect() throws Exception { restateChangeLog.setAspectName(aspectName1); restateChangeLog.setCreated(TEST_AUDIT_STAMP); restateChangeLog.setAspect(aspect); - restateChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1)); + restateChangeLog.setSystemMetadata(futureSystemMetadata); restateChangeLog.setPreviousAspectValue(aspect); restateChangeLog.setPreviousSystemMetadata( - simulatePullFromDB(initialSystemMetadata, SystemMetadata.class)); + simulatePullFromDB(futureSystemMetadata, SystemMetadata.class)); restateChangeLog.setEntityKeyAspect( GenericRecordUtils.serializeAspect( EntityKeyUtils.convertUrnToEntityKey( @@ -636,11 +648,7 @@ public void testReingestLineageAspect() throws Exception { clearInvocations(_mockProducer); _entityServiceImpl.ingestAspects( - opContext, - entityUrn, - pairToIngest, - TEST_AUDIT_STAMP, - AspectGenerationUtils.createSystemMetadata()); + opContext, entityUrn, pairToIngest, TEST_AUDIT_STAMP, futureSystemMetadata); verify(_mockProducer, times(1)) .produceMetadataChangeLog( @@ -682,6 +690,12 @@ public void testReingestLineageProposal() throws Exception { initialChangeLog.setAspect(genericAspect); initialChangeLog.setSystemMetadata(metadata1); + SystemMetadata futureSystemMetadata = AspectGenerationUtils.createSystemMetadata(1); + futureSystemMetadata.setLastObserved(futureSystemMetadata.getLastObserved() + 1); + + MetadataChangeProposal mcp2 = new MetadataChangeProposal(mcp1.data().copy()); + mcp2.getSystemMetadata().setLastObserved(futureSystemMetadata.getLastObserved()); + final MetadataChangeLog restateChangeLog = new MetadataChangeLog(); restateChangeLog.setEntityType(entityUrn.getEntityType()); restateChangeLog.setEntityUrn(entityUrn); @@ -689,9 +703,10 @@ public void testReingestLineageProposal() throws Exception { restateChangeLog.setAspectName(aspectName1); restateChangeLog.setCreated(TEST_AUDIT_STAMP); restateChangeLog.setAspect(genericAspect); - restateChangeLog.setSystemMetadata(AspectGenerationUtils.createSystemMetadata(1)); + restateChangeLog.setSystemMetadata(futureSystemMetadata); restateChangeLog.setPreviousAspectValue(genericAspect); - restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class)); + restateChangeLog.setPreviousSystemMetadata( + simulatePullFromDB(futureSystemMetadata, SystemMetadata.class)); Map latestAspects = _entityServiceImpl.getLatestAspectsForUrn( @@ -706,7 +721,7 @@ public void testReingestLineageProposal() throws Exception { // unless invocations are cleared clearInvocations(_mockProducer); - _entityServiceImpl.ingestProposal(opContext, mcp1, TEST_AUDIT_STAMP, false); + _entityServiceImpl.ingestProposal(opContext, mcp2, TEST_AUDIT_STAMP, false); verify(_mockProducer, times(1)) .produceMetadataChangeLog( @@ -1390,7 +1405,7 @@ public void testIngestSameAspect() throws AssertionError { SystemMetadata metadata1 = AspectGenerationUtils.createSystemMetadata(1625792689, "run-123"); SystemMetadata metadata2 = AspectGenerationUtils.createSystemMetadata(1635792689, "run-456"); SystemMetadata metadata3 = - AspectGenerationUtils.createSystemMetadata(1635792689, "run-123", "run-456", "1"); + AspectGenerationUtils.createSystemMetadata(1635792689, "run-456", "run-123", "1"); List items = List.of( @@ -1482,6 +1497,9 @@ public void testIngestSameAspect() throws AssertionError { assertTrue( DataTemplateUtil.areEqual( + EntityApiUtils.parseSystemMetadata(readAspectDao2.getSystemMetadata()), metadata3), + String.format( + "Expected %s == %s", EntityApiUtils.parseSystemMetadata(readAspectDao2.getSystemMetadata()), metadata3)); verify(_mockProducer, times(0)) @@ -2179,6 +2197,474 @@ public void testExists() throws Exception { Set.of(existentUrn, noStatusUrn, softDeletedUrn)); } + @Test + public void testBatchDuplicate() throws Exception { + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:batchDuplicateTest"); + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + ChangeItemImpl item1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(STATUS_ASPECT_NAME) + .recordTemplate(new Status().setRemoved(true)) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + ChangeItemImpl item2 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(STATUS_ASPECT_NAME) + .recordTemplate(new Status().setRemoved(false)) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(item1, item2)) + .build(), + false, + true); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 2); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, CORP_USER_ENTITY_NAME, entityUrn, STATUS_ASPECT_NAME); + assertEquals( + envelopedAspect.getSystemMetadata().getVersion(), + "2", + "Expected version 2 after accounting for sequential duplicates"); + assertEquals( + envelopedAspect.getValue().toString(), + "{removed=false}", + "Expected 2nd item to be the latest"); + } + + @Test + public void testBatchPatchWithTrailingNoOp() throws Exception { + Urn entityUrn = + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchWithTrailingNoOp,PROD)"); + TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); + Urn tag2 = UrnUtils.getUrn("urn:li:tag:tag2"); + Urn tagOther = UrnUtils.getUrn("urn:li:tag:other"); + + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + + ChangeItemImpl initialAspectTag1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .recordTemplate( + new GlobalTags() + .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1)))) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + + PatchItemImpl patchAdd2 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + PatchItemImpl patchRemoveNonExistent = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.REMOVE, tagOther))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + // establish base entity + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(initialAspectTag1)) + .build(), + false, + true); + + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchAdd2, patchRemoveNonExistent)) + .build(), + false, + true); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals( + envelopedAspect.getSystemMetadata().getVersion(), + "2", + "Expected version 3. 1 - Initial, + 1 add, 1 remove"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Set.of(tag1, tag2), + "Expected both tags"); + } + + @Test + public void testBatchPatchAdd() throws Exception { + Urn entityUrn = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)"); + TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); + TagUrn tag2 = TagUrn.createFromString("urn:li:tag:tag2"); + TagUrn tag3 = TagUrn.createFromString("urn:li:tag:tag3"); + + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + + ChangeItemImpl initialAspectTag1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .recordTemplate( + new GlobalTags() + .setTags(new TagAssociationArray(new TagAssociation().setTag(tag1)))) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + + PatchItemImpl patchAdd3 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag3))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + PatchItemImpl patchAdd2 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + PatchItemImpl patchAdd1 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag1))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + // establish base entity + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(initialAspectTag1)) + .build(), + false, + true); + + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchAdd3, patchAdd2, patchAdd1)) + .build(), + false, + true); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "3", "Expected version 4"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Set.of(tag1, tag2, tag3), + "Expected all tags"); + } + + @Test + public void testBatchPatchAddDuplicate() throws Exception { + Urn entityUrn = + UrnUtils.getUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,testBatchPatchAdd,PROD)"); + List initialTags = + List.of( + TagUrn.createFromString("urn:li:tag:__default_large_table"), + TagUrn.createFromString("urn:li:tag:__default_low_queries"), + TagUrn.createFromString("urn:li:tag:__default_low_changes"), + TagUrn.createFromString("urn:li:tag:!10TB+ tables")) + .stream() + .map(tag -> new TagAssociation().setTag(tag)) + .collect(Collectors.toList()); + TagUrn tag2 = TagUrn.createFromString("urn:li:tag:$ 1TB+"); + + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + + SystemMetadata patchSystemMetadata = new SystemMetadata(); + patchSystemMetadata.setLastObserved(systemMetadata.getLastObserved() + 1); + patchSystemMetadata.setProperties(new StringMap(Map.of(APP_SOURCE, METADATA_TESTS_SOURCE))); + + ChangeItemImpl initialAspectTag1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .recordTemplate(new GlobalTags().setTags(new TagAssociationArray(initialTags))) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + + PatchItemImpl patchAdd2 = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag2))) + .build() + .getJsonPatch()) + .systemMetadata(patchSystemMetadata) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + // establish base entity + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(initialAspectTag1)) + .build(), + false, + true); + + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchAdd2, patchAdd2)) // duplicate + .build(), + false, + true); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "2", "Expected version 2"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Stream.concat(initialTags.stream().map(TagAssociation::getTag), Stream.of(tag2)) + .collect(Collectors.toSet()), + "Expected all tags"); + } + + @Test + public void testPatchRemoveNonExistent() throws Exception { + Urn entityUrn = + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,testPatchRemoveNonExistent,PROD)"); + TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); + + PatchItemImpl patchRemove = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.REMOVE, tag1))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + List results = + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchRemove)) + .build(), + false, + true); + + assertEquals(results.size(), 4, "Expected default aspects + empty globalTags"); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "1", "Expected version 4"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Set.of(), + "Expected empty tags"); + } + + @Test + public void testPatchAddNonExistent() throws Exception { + Urn entityUrn = + UrnUtils.getUrn( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,testPatchAddNonExistent,PROD)"); + TagUrn tag1 = TagUrn.createFromString("urn:li:tag:tag1"); + + PatchItemImpl patchAdd = + PatchItemImpl.builder() + .urn(entityUrn) + .entitySpec(_testEntityRegistry.getEntitySpec(DATASET_ENTITY_NAME)) + .aspectName(GLOBAL_TAGS_ASPECT_NAME) + .aspectSpec( + _testEntityRegistry + .getEntitySpec(DATASET_ENTITY_NAME) + .getAspectSpec(GLOBAL_TAGS_ASPECT_NAME)) + .patch( + GenericJsonPatch.builder() + .arrayPrimaryKeys(Map.of("properties", List.of("tag"))) + .patch(List.of(tagPatchOp(PatchOperationType.ADD, tag1))) + .build() + .getJsonPatch()) + .auditStamp(AuditStampUtils.createDefaultAuditStamp()) + .build(_testEntityRegistry); + + List results = + _entityServiceImpl.ingestAspects( + opContext, + AspectsBatchImpl.builder() + .retrieverContext(opContext.getRetrieverContext().get()) + .items(List.of(patchAdd)) + .build(), + false, + true); + + assertEquals(results.size(), 4, "Expected default aspects + globalTags"); + + // List aspects urns + ListUrnsResult batch = _entityServiceImpl.listUrns(opContext, entityUrn.getEntityType(), 0, 1); + + assertEquals(batch.getStart().intValue(), 0); + assertEquals(batch.getCount().intValue(), 1); + assertEquals(batch.getTotal().intValue(), 1); + assertEquals(batch.getEntities().size(), 1); + assertEquals(entityUrn.toString(), batch.getEntities().get(0).toString()); + + EnvelopedAspect envelopedAspect = + _entityServiceImpl.getLatestEnvelopedAspect( + opContext, DATASET_ENTITY_NAME, entityUrn, GLOBAL_TAGS_ASPECT_NAME); + assertEquals(envelopedAspect.getSystemMetadata().getVersion(), "1", "Expected version 4"); + assertEquals( + new GlobalTags(envelopedAspect.getValue().data()) + .getTags().stream().map(TagAssociation::getTag).collect(Collectors.toSet()), + Set.of(tag1), + "Expected all tags"); + } + @Nonnull protected com.linkedin.entity.Entity createCorpUserEntity(Urn entityUrn, String email) throws Exception { @@ -2210,4 +2696,14 @@ protected Pair getAspectRecor RecordUtils.toRecordTemplate(clazz, objectMapper.writeValueAsString(aspect)); return new Pair<>(AspectGenerationUtils.getAspectName(aspect), recordTemplate); } + + private static GenericJsonPatch.PatchOp tagPatchOp(PatchOperationType op, Urn tagUrn) { + GenericJsonPatch.PatchOp patchOp = new GenericJsonPatch.PatchOp(); + patchOp.setOp(op.getValue()); + patchOp.setPath(String.format("/tags/%s", tagUrn)); + if (PatchOperationType.ADD.equals(op)) { + patchOp.setValue(Map.of("tag", tagUrn.toString())); + } + return patchOp; + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImplTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImplTest.java new file mode 100644 index 00000000000000..3f6b301e72aa5a --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/ebean/batch/ChangeItemImplTest.java @@ -0,0 +1,41 @@ +package com.linkedin.metadata.entity.ebean.batch; + +import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME; +import static org.testng.Assert.assertFalse; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.AspectGenerationUtils; +import com.linkedin.mxe.SystemMetadata; +import io.datahubproject.test.metadata.context.TestOperationContexts; +import org.testng.annotations.Test; + +public class ChangeItemImplTest { + private static final AuditStamp TEST_AUDIT_STAMP = AspectGenerationUtils.createAuditStamp(); + + @Test + public void testBatchDuplicate() throws Exception { + Urn entityUrn = UrnUtils.getUrn("urn:li:corpuser:batchDuplicateTest"); + SystemMetadata systemMetadata = AspectGenerationUtils.createSystemMetadata(); + ChangeItemImpl item1 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(STATUS_ASPECT_NAME) + .recordTemplate(new Status().setRemoved(true)) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + ChangeItemImpl item2 = + ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(STATUS_ASPECT_NAME) + .recordTemplate(new Status().setRemoved(false)) + .systemMetadata(systemMetadata.copy()) + .auditStamp(TEST_AUDIT_STAMP) + .build(TestOperationContexts.emptyAspectRetriever(null)); + + assertFalse(item1.isDatabaseDuplicateOf(item2)); + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java b/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java index 6139776702c715..1661f5f02ee593 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/schemafields/sideeffects/SchemaFieldSideEffectTest.java @@ -151,7 +151,7 @@ public void schemaMetadataToSchemaFieldKeyTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) .aspectName(SCHEMA_FIELD_ALIASES_ASPECT) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY @@ -172,7 +172,7 @@ public void schemaMetadataToSchemaFieldKeyTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)")) .aspectName(SCHEMA_FIELD_ALIASES_ASPECT) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY @@ -248,7 +248,7 @@ public void statusToSchemaFieldStatusTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) .aspectName(STATUS_ASPECT_NAME) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY @@ -263,7 +263,7 @@ public void statusToSchemaFieldStatusTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)")) .aspectName(STATUS_ASPECT_NAME) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY @@ -324,7 +324,7 @@ public void statusToSchemaFieldStatusTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) .aspectName(STATUS_ASPECT_NAME) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY @@ -339,7 +339,7 @@ public void statusToSchemaFieldStatusTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)")) .aspectName(STATUS_ASPECT_NAME) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY @@ -354,7 +354,7 @@ public void statusToSchemaFieldStatusTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) .aspectName(SCHEMA_FIELD_ALIASES_ASPECT) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY @@ -375,7 +375,7 @@ public void statusToSchemaFieldStatusTest() { UrnUtils.getUrn( "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_name)")) .aspectName(SCHEMA_FIELD_ALIASES_ASPECT) - .changeType(changeType) + .changeType(ChangeType.UPSERT) .entitySpec(TEST_REGISTRY.getEntitySpec(SCHEMA_FIELD_ENTITY_NAME)) .aspectSpec( TEST_REGISTRY diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 4945b36a251c26..15cd126408a7cc 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -159,7 +159,7 @@ ebean: autoCreateDdl: ${EBEAN_AUTOCREATE:false} postgresUseIamAuth: ${EBEAN_POSTGRES_USE_AWS_IAM_AUTH:false} locking: - enabled: ${EBEAN_LOCKING_ENABLED:true} + enabled: ${EBEAN_LOCKING_ENABLED:false} durationSeconds: ${EBEAN_LOCKING_DURATION_SECONDS:60} maximumLocks: ${EBEAN_LOCKING_MAXIMUM_LOCKS:20000} diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java index fafca9b1139731..993edc44daeff1 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/GenericRecordUtils.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.common.urn.Urn; import com.linkedin.data.ByteString; +import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; @@ -13,6 +14,8 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.mxe.GenericAspect; import com.linkedin.mxe.GenericPayload; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.stream.Collectors; @@ -23,6 +26,22 @@ public class GenericRecordUtils { private GenericRecordUtils() {} + public static T copy(T input, Class clazz) { + try { + if (input == null) { + return null; + } + Constructor constructor = clazz.getConstructor(DataMap.class); + return constructor.newInstance(input.data().copy()); + } catch (CloneNotSupportedException + | InvocationTargetException + | NoSuchMethodException + | InstantiationException + | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + /** Deserialize the given value into the aspect based on the input aspectSpec */ @Nonnull public static RecordTemplate deserializeAspect( From 61fffb2a81eb958cedbca29f6cc53091efe00f0e Mon Sep 17 00:00:00 2001 From: Chakru <161002324+chakru-r@users.noreply.github.com> Date: Tue, 10 Dec 2024 19:57:11 +0530 Subject: [PATCH 160/174] build(coverage): rename python coverage reports (#12071) --- gradle/coverage/python-coverage.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/coverage/python-coverage.gradle b/gradle/coverage/python-coverage.gradle index 0ab921dfb21ffd..23d6e37387ed83 100644 --- a/gradle/coverage/python-coverage.gradle +++ b/gradle/coverage/python-coverage.gradle @@ -1,7 +1,7 @@ //coverage related args to be passed to pytest ext.get_coverage_args = { test_name = "" -> - def coverage_file_name = "pycov-${project.name}${test_name}.xml" + def coverage_file_name = "coverage-${project.name}${test_name}.xml" /* Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one folder From 57b12bd9cb9689638a34932b239540981f95fd6d Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware <159135491+sagar-salvi-apptware@users.noreply.github.com> Date: Tue, 10 Dec 2024 22:06:01 +0530 Subject: [PATCH 161/174] fix(ingest): replace sqllineage/sqlparse with our SQL parser (#12020) --- docs/how/updating-datahub.md | 23 ++- metadata-ingestion-modules/gx-plugin/setup.py | 12 +- .../gx-plugin/src/datahub_gx_plugin/action.py | 24 ++- .../docs/sources/redash/redash.md | 7 +- metadata-ingestion/setup.py | 23 +-- .../src/datahub/ingestion/source/mode.py | 23 --- .../src/datahub/ingestion/source/redash.py | 76 ++------- .../datahub/ingestion/source/unity/usage.py | 31 ++-- .../utilities/sql_lineage_parser_impl.py | 160 ------------------ .../src/datahub/utilities/sql_parser.py | 94 ---------- .../src/datahub/utilities/sql_parser_base.py | 21 --- .../tests/unit/test_redash_source.py | 6 +- .../tests/unit/utilities/test_utilities.py | 65 +++++-- 13 files changed, 130 insertions(+), 435 deletions(-) delete mode 100644 metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py delete mode 100644 metadata-ingestion/src/datahub/utilities/sql_parser.py delete mode 100644 metadata-ingestion/src/datahub/utilities/sql_parser_base.py diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index bcc89332cc1c1b..d8fe06abad6252 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -19,19 +19,21 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ## Next - #11560 - The PowerBI ingestion source configuration option include_workspace_name_in_dataset_urn determines whether the workspace name is included in the PowerBI dataset's URN.
PowerBI allows to have identical name of semantic model and their tables across the workspace, It will overwrite the semantic model in-case of multi-workspace ingestion.
- Entity urn with `include_workspace_name_in_dataset_urn: false` - ``` - urn:li:dataset:(urn:li:dataPlatform:powerbi,[.].,) - ``` + Entity urn with `include_workspace_name_in_dataset_urn: false` - Entity urn with `include_workspace_name_in_dataset_urn: true` - ``` - urn:li:dataset:(urn:li:dataPlatform:powerbi,[.]...,) - ``` + ``` + urn:li:dataset:(urn:li:dataPlatform:powerbi,[.].,) + ``` + + Entity urn with `include_workspace_name_in_dataset_urn: true` + + ``` + urn:li:dataset:(urn:li:dataPlatform:powerbi,[.]...,) + ``` The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatiblity, However, we recommend enabling this flag after performing the necessary cleanup. If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI: - `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true. + `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true. - #11701: The Fivetran `sources_to_database` field is deprecated in favor of setting directly within `sources_to_platform_instance..database`. - #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information. @@ -48,6 +50,9 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11619 - schema field/column paths can no longer be duplicated within the schema - #11570 - The `DatahubClientConfig`'s server field no longer defaults to `http://localhost:8080`. Be sure to explicitly set this. - #11570 - If a `datahub_api` is explicitly passed to a stateful ingestion config provider, it will be used. We previously ignored it if the pipeline context also had a graph object. +- #11518 - DataHub Garbage Collection: Various entities that are soft-deleted (after 10d) or are timeseries _entities_ (dataprocess, execution requests) will be removed automatically using logic in the `datahub-gc` ingestion source. +- #12020 - Removed `sql_parser` configuration from the Redash source, as Redash now exclusively uses the sqlglot-based parser for lineage extraction. +- #12020 - Removed `datahub.utilities.sql_parser`, `datahub.utilities.sql_parser_base` and `datahub.utilities.sql_lineage_parser_impl` module along with `SqlLineageSQLParser` and `DefaultSQLParser`. Use `create_lineage_sql_parsed_result` from `datahub.sql_parsing.sqlglot_lineage` module instead. - #11518 - DataHub Garbage Collection: Various entities that are soft-deleted (after 10d) or are timeseries *entities* (dataprocess, execution requests) will be removed automatically using logic in the `datahub-gc` ingestion diff --git a/metadata-ingestion-modules/gx-plugin/setup.py b/metadata-ingestion-modules/gx-plugin/setup.py index e87bbded96584e..73d5d1a9a02f18 100644 --- a/metadata-ingestion-modules/gx-plugin/setup.py +++ b/metadata-ingestion-modules/gx-plugin/setup.py @@ -15,15 +15,6 @@ def get_long_description(): rest_common = {"requests", "requests_file"} -# TODO: Can we move away from sqllineage and use sqlglot ?? -sqllineage_lib = { - "sqllineage==1.3.8", - # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage. - # There have previously been issues from not pinning sqlparse, so it's best to pin it. - # Related: https://github.com/reata/sqllineage/issues/361 and https://github.com/reata/sqllineage/pull/360 - "sqlparse==0.4.4", -} - _version: str = package_metadata["__version__"] _self_pin = ( f"=={_version}" @@ -43,8 +34,7 @@ def get_long_description(): # https://github.com/ipython/traitlets/issues/741 "traitlets<5.2.2", *rest_common, - *sqllineage_lib, - f"acryl-datahub[datahub-rest]{_self_pin}", + f"acryl-datahub[datahub-rest,sql-parser]{_self_pin}", } mypy_stubs = { diff --git a/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py b/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py index 2ad301a38d0028..2d89d26997d1f3 100644 --- a/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py +++ b/metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/action.py @@ -34,8 +34,9 @@ ) from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance from datahub.metadata.schema_classes import PartitionSpecClass, PartitionTypeClass +from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result from datahub.utilities._markupsafe_compat import MARKUPSAFE_PATCHED -from datahub.utilities.sql_parser import DefaultSQLParser +from datahub.utilities.urns.dataset_urn import DatasetUrn from great_expectations.checkpoint.actions import ValidationAction from great_expectations.core.batch import Batch from great_expectations.core.batch_spec import ( @@ -677,10 +678,23 @@ def get_dataset_partitions(self, batch_identifier, data_asset): query=query, customProperties=batchSpecProperties, ) - try: - tables = DefaultSQLParser(query).get_tables() - except Exception as e: - logger.warning(f"Sql parser failed on {query} with {e}") + + data_platform = get_platform_from_sqlalchemy_uri(str(sqlalchemy_uri)) + sql_parser_in_tables = create_lineage_sql_parsed_result( + query=query, + platform=data_platform, + env=self.env, + platform_instance=None, + default_db=None, + ) + tables = [ + DatasetUrn.from_string(table_urn).name + for table_urn in sql_parser_in_tables.in_tables + ] + if sql_parser_in_tables.debug_info.table_error: + logger.warning( + f"Sql parser failed on {query} with {sql_parser_in_tables.debug_info.table_error}" + ) tables = [] if len(set(tables)) != 1: diff --git a/metadata-ingestion/docs/sources/redash/redash.md b/metadata-ingestion/docs/sources/redash/redash.md index 8f8c5c85496a09..f23a523cebc913 100644 --- a/metadata-ingestion/docs/sources/redash/redash.md +++ b/metadata-ingestion/docs/sources/redash/redash.md @@ -1,5 +1,2 @@ -Note! The integration can use an SQL parser to try to parse the tables the chart depends on. This parsing is disabled by default, -but can be enabled by setting `parse_table_names_from_sql: true`. The default parser is based on the [`sqllineage`](https://pypi.org/project/sqllineage/) package. -As this package doesn't officially support all the SQL dialects that Redash supports, the result might not be correct. You can, however, implement a -custom parser and take it into use by setting the `sql_parser` configuration value. A custom SQL parser must inherit from `datahub.utilities.sql_parser.SQLParser` -and must be made available to Datahub by ,for example, installing it. The configuration then needs to be set to `module_name.ClassName` of the parser. +Note! The integration can use an SQL parser to try to parse the tables the chart depends on. This parsing is disabled by default, +but can be enabled by setting `parse_table_names_from_sql: true`. The parser is based on the [`sqlglot`](https://pypi.org/project/sqlglot/) package. diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 5ae5438e212c5b..415871d30175f8 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -159,14 +159,6 @@ | classification_lib ) -sqllineage_lib = { - "sqllineage==1.3.8", - # We don't have a direct dependency on sqlparse but it is a dependency of sqllineage. - # There have previously been issues from not pinning sqlparse, so it's best to pin it. - # Related: https://github.com/reata/sqllineage/issues/361 and https://github.com/reata/sqllineage/pull/360 - "sqlparse==0.4.4", -} - aws_common = { # AWS Python SDK "boto3", @@ -216,7 +208,6 @@ "sqlalchemy-redshift>=0.8.3", "GeoAlchemy2", "redshift-connector>=2.1.0", - *sqllineage_lib, *path_spec_common, } @@ -464,9 +455,7 @@ # It's technically wrong for packages to depend on setuptools. However, it seems mlflow does it anyways. "setuptools", }, - "mode": {"requests", "python-liquid", "tenacity>=8.0.1"} - | sqllineage_lib - | sqlglot_lib, + "mode": {"requests", "python-liquid", "tenacity>=8.0.1"} | sqlglot_lib, "mongodb": {"pymongo[srv]>=3.11", "packaging"}, "mssql": sql_common | mssql_common, "mssql-odbc": sql_common | mssql_common | {"pyodbc"}, @@ -482,7 +471,7 @@ | pyhive_common | {"psycopg2-binary", "pymysql>=1.0.2"}, "pulsar": {"requests"}, - "redash": {"redash-toolbelt", "sql-metadata"} | sqllineage_lib, + "redash": {"redash-toolbelt", "sql-metadata"} | sqlglot_lib, "redshift": sql_common | redshift_common | usage_common @@ -503,9 +492,7 @@ "slack": slack, "superset": superset_common, "preset": superset_common, - # FIXME: I don't think tableau uses sqllineage anymore so we should be able - # to remove that dependency. - "tableau": {"tableauserverclient>=0.24.0"} | sqllineage_lib | sqlglot_lib, + "tableau": {"tableauserverclient>=0.24.0"} | sqlglot_lib, "teradata": sql_common | usage_common | sqlglot_lib @@ -527,9 +514,9 @@ ), "powerbi-report-server": powerbi_report_server, "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.8.2"}, - "unity-catalog": databricks | sql_common | sqllineage_lib, + "unity-catalog": databricks | sql_common, # databricks is alias for unity-catalog and needs to be kept in sync - "databricks": databricks | sql_common | sqllineage_lib, + "databricks": databricks | sql_common, "fivetran": snowflake_common | bigquery_common | sqlglot_lib, "qlik-sense": sqlglot_lib | {"requests", "websocket-client"}, "sigma": sqlglot_lib | {"requests"}, diff --git a/metadata-ingestion/src/datahub/ingestion/source/mode.py b/metadata-ingestion/src/datahub/ingestion/source/mode.py index e24cba9b193d31..c1ab9271ce13ae 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mode.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mode.py @@ -18,7 +18,6 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import ConnectionError from requests.models import HTTPBasicAuth, HTTPError -from sqllineage.runner import LineageRunner from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential import datahub.emitter.mce_builder as builder @@ -820,28 +819,6 @@ def _get_definition(self, definition_name): ) return None - @lru_cache(maxsize=None) - def _get_source_from_query(self, raw_query: str) -> set: - query = self._replace_definitions(raw_query) - parser = LineageRunner(query) - source_paths = set() - try: - for table in parser.source_tables: - sources = str(table).split(".") - source_schema, source_table = sources[-2], sources[-1] - if source_schema == "": - source_schema = str(self.config.default_schema) - - source_paths.add(f"{source_schema}.{source_table}") - except Exception as e: - self.report.report_failure( - title="Failed to Extract Lineage From Query", - message="Unable to retrieve lineage from Mode query.", - context=f"Query: {raw_query}, Error: {str(e)}", - ) - - return source_paths - def _get_datasource_urn( self, platform: str, diff --git a/metadata-ingestion/src/datahub/ingestion/source/redash.py b/metadata-ingestion/src/datahub/ingestion/source/redash.py index 581e32d29dceaf..f11d1944029ebb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/redash.py +++ b/metadata-ingestion/src/datahub/ingestion/source/redash.py @@ -2,7 +2,7 @@ import math import sys from dataclasses import dataclass, field -from typing import Dict, Iterable, List, Optional, Set, Type +from typing import Dict, Iterable, List, Optional, Set import dateutil.parser as dp from packaging import version @@ -22,7 +22,6 @@ platform_name, support_status, ) -from datahub.ingestion.api.registry import import_path from datahub.ingestion.api.source import Source, SourceCapability, SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import ( @@ -39,9 +38,9 @@ ChartTypeClass, DashboardInfoClass, ) +from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result from datahub.utilities.lossy_collections import LossyDict, LossyList from datahub.utilities.perf_timer import PerfTimer -from datahub.utilities.sql_parser_base import SQLParser from datahub.utilities.threaded_iterator_executor import ThreadedIteratorExecutor logger = logging.getLogger(__name__) @@ -270,10 +269,6 @@ class RedashConfig(ConfigModel): parse_table_names_from_sql: bool = Field( default=False, description="See note below." ) - sql_parser: str = Field( - default="datahub.utilities.sql_parser.DefaultSQLParser", - description="custom SQL parser. See note below for details.", - ) env: str = Field( default=DEFAULT_ENV, @@ -354,7 +349,6 @@ def __init__(self, ctx: PipelineContext, config: RedashConfig): self.api_page_limit = self.config.api_page_limit or math.inf self.parse_table_names_from_sql = self.config.parse_table_names_from_sql - self.sql_parser_path = self.config.sql_parser logger.info( f"Running Redash ingestion with parse_table_names_from_sql={self.parse_table_names_from_sql}" @@ -380,31 +374,6 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> Source: config = RedashConfig.parse_obj(config_dict) return cls(ctx, config) - @classmethod - def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]: - assert "." in sql_parser_path, "sql_parser-path must contain a ." - parser_cls = import_path(sql_parser_path) - - if not issubclass(parser_cls, SQLParser): - raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}") - return parser_cls - - @classmethod - def _get_sql_table_names(cls, sql: str, sql_parser_path: str) -> List[str]: - parser_cls = cls._import_sql_parser_cls(sql_parser_path) - - try: - sql_table_names: List[str] = parser_cls(sql).get_tables() - except Exception as e: - logger.warning(f"Sql parser failed on {sql} with {e}") - return [] - - # Remove quotes from table names - sql_table_names = [t.replace('"', "") for t in sql_table_names] - sql_table_names = [t.replace("`", "") for t in sql_table_names] - - return sql_table_names - def _get_chart_data_source(self, data_source_id: Optional[int] = None) -> Dict: url = f"/api/data_sources/{data_source_id}" resp = self.client._get(url).json() @@ -441,14 +410,6 @@ def _get_database_name_based_on_datasource( return database_name - def _construct_datalineage_urn( - self, platform: str, database_name: str, sql_table_name: str - ) -> str: - full_dataset_name = get_full_qualified_name( - platform, database_name, sql_table_name - ) - return builder.make_dataset_urn(platform, full_dataset_name, self.config.env) - def _get_datasource_urns( self, data_source: Dict, sql_query_data: Dict = {} ) -> Optional[List[str]]: @@ -464,34 +425,23 @@ def _get_datasource_urns( # Getting table lineage from SQL parsing if self.parse_table_names_from_sql and data_source_syntax == "sql": dataset_urns = list() - try: - sql_table_names = self._get_sql_table_names( - query, self.sql_parser_path - ) - except Exception as e: + sql_parser_in_tables = create_lineage_sql_parsed_result( + query=query, + platform=platform, + env=self.config.env, + platform_instance=None, + default_db=database_name, + ) + # make sure dataset_urns is not empty list + dataset_urns = sql_parser_in_tables.in_tables + if sql_parser_in_tables.debug_info.table_error: self.report.queries_problem_parsing.add(str(query_id)) self.error( logger, "sql-parsing", - f"exception {e} in parsing query-{query_id}-datasource-{data_source_id}", + f"exception {sql_parser_in_tables.debug_info.table_error} in parsing query-{query_id}-datasource-{data_source_id}", ) - sql_table_names = [] - for sql_table_name in sql_table_names: - try: - dataset_urns.append( - self._construct_datalineage_urn( - platform, database_name, sql_table_name - ) - ) - except Exception: - self.report.queries_problem_parsing.add(str(query_id)) - self.warn( - logger, - "data-urn-invalid", - f"Problem making URN for {sql_table_name} parsed from query {query_id}", - ) - # make sure dataset_urns is not empty list return dataset_urns if len(dataset_urns) > 0 else None else: diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py index 8c42ac81b98cf5..718818d9b347bf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/usage.py @@ -7,7 +7,6 @@ import pyspark from databricks.sdk.service.sql import QueryStatementType -from sqllineage.runner import LineageRunner from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.source_helpers import auto_empty_dataset_usage_statistics @@ -22,7 +21,9 @@ from datahub.ingestion.source.unity.report import UnityCatalogReport from datahub.ingestion.source.usage.usage_common import UsageAggregator from datahub.metadata.schema_classes import OperationClass +from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result from datahub.sql_parsing.sqlglot_utils import get_query_fingerprint +from datahub.utilities.urns.dataset_urn import DatasetUrn logger = logging.getLogger(__name__) @@ -48,6 +49,7 @@ class UnityCatalogUsageExtractor: proxy: UnityCatalogApiProxy table_urn_builder: Callable[[TableReference], str] user_urn_builder: Callable[[str], str] + platform: str = "databricks" def __post_init__(self): self.usage_aggregator = UsageAggregator[TableReference](self.config) @@ -173,7 +175,7 @@ def _parse_query( self, query: Query, table_map: TableMap ) -> Optional[QueryTableInfo]: with self.report.usage_perf_report.sql_parsing_timer: - table_info = self._parse_query_via_lineage_runner(query.query_text) + table_info = self._parse_query_via_sqlglot(query.query_text) if table_info is None and query.statement_type == QueryStatementType.SELECT: with self.report.usage_perf_report.spark_sql_parsing_timer: table_info = self._parse_query_via_spark_sql_plan(query.query_text) @@ -191,26 +193,33 @@ def _parse_query( ), ) - def _parse_query_via_lineage_runner(self, query: str) -> Optional[StringTableInfo]: + def _parse_query_via_sqlglot(self, query: str) -> Optional[StringTableInfo]: try: - runner = LineageRunner(query) + sql_parser_in_tables = create_lineage_sql_parsed_result( + query=query, + default_db=None, + platform=self.platform, + env=self.config.env, + platform_instance=None, + ) + return GenericTableInfo( source_tables=[ - self._parse_sqllineage_table(table) - for table in runner.source_tables + self._parse_sqlglot_table(table) + for table in sql_parser_in_tables.in_tables ], target_tables=[ - self._parse_sqllineage_table(table) - for table in runner.target_tables + self._parse_sqlglot_table(table) + for table in sql_parser_in_tables.out_tables ], ) except Exception as e: - logger.info(f"Could not parse query via lineage runner, {query}: {e!r}") + logger.info(f"Could not parse query via sqlglot, {query}: {e!r}") return None @staticmethod - def _parse_sqllineage_table(sqllineage_table: object) -> str: - full_table_name = str(sqllineage_table) + def _parse_sqlglot_table(table_urn: str) -> str: + full_table_name = DatasetUrn.from_string(table_urn).name default_schema = "." if full_table_name.startswith(default_schema): return full_table_name[len(default_schema) :] diff --git a/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py b/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py deleted file mode 100644 index 5a8802c7a0a49c..00000000000000 --- a/metadata-ingestion/src/datahub/utilities/sql_lineage_parser_impl.py +++ /dev/null @@ -1,160 +0,0 @@ -import contextlib -import logging -import re -import unittest -import unittest.mock -from typing import Dict, List, Optional, Set - -from sqllineage.core.holders import Column, SQLLineageHolder -from sqllineage.exceptions import SQLLineageException - -from datahub.utilities.sql_parser_base import SQLParser, SqlParserException - -with contextlib.suppress(ImportError): - import sqlparse - from networkx import DiGraph - from sqllineage.core import LineageAnalyzer - - import datahub.utilities.sqllineage_patch -logger = logging.getLogger(__name__) - - -class SqlLineageSQLParserImpl(SQLParser): - _DATE_SWAP_TOKEN = "__d_a_t_e" - _HOUR_SWAP_TOKEN = "__h_o_u_r" - _TIMESTAMP_SWAP_TOKEN = "__t_i_m_e_s_t_a_m_p" - _DATA_SWAP_TOKEN = "__d_a_t_a" - _ADMIN_SWAP_TOKEN = "__a_d_m_i_n" - _MYVIEW_SQL_TABLE_NAME_TOKEN = "__my_view__.__sql_table_name__" - _MYVIEW_LOOKER_TOKEN = "my_view.SQL_TABLE_NAME" - - def __init__(self, sql_query: str, use_raw_names: bool = False) -> None: - super().__init__(sql_query) - original_sql_query = sql_query - self._use_raw_names = use_raw_names - - # SqlLineageParser makes mistakes on lateral flatten queries, use the prefix - if "lateral flatten" in sql_query: - sql_query = sql_query[: sql_query.find("lateral flatten")] - - # Replace reserved words that break SqlLineageParser - self.token_to_original: Dict[str, str] = { - self._DATE_SWAP_TOKEN: "date", - self._HOUR_SWAP_TOKEN: "hour", - self._TIMESTAMP_SWAP_TOKEN: "timestamp", - self._DATA_SWAP_TOKEN: "data", - self._ADMIN_SWAP_TOKEN: "admin", - } - for replacement, original in self.token_to_original.items(): - # Replace original tokens with replacement. Since table and column name can contain a hyphen('-'), - # also prevent original tokens appearing as part of these names with a hyphen from getting substituted. - sql_query = re.sub( - rf"((? List[str]: - result: List[str] = [] - if self._sql_holder is None: - logger.error("sql holder not present so cannot get tables") - return result - for table in self._sql_holder.source_tables: - table_normalized = re.sub( - r"^.", - "", - ( - str(table) - if not self._use_raw_names - else f"{table.schema.raw_name}.{table.raw_name}" - ), - ) - result.append(str(table_normalized)) - - # We need to revert TOKEN replacements - for token, replacement in self.token_to_original.items(): - result = [replacement if c == token else c for c in result] - result = [ - self._MYVIEW_LOOKER_TOKEN if c == self._MYVIEW_SQL_TABLE_NAME_TOKEN else c - for c in result - ] - - # Sort tables to make the list deterministic - result.sort() - - return result - - def get_columns(self) -> List[str]: - if self._sql_holder is None: - raise SqlParserException("sql holder not present so cannot get columns") - graph: DiGraph = self._sql_holder.graph # For mypy attribute checking - column_nodes = [n for n in graph.nodes if isinstance(n, Column)] - column_graph = graph.subgraph(column_nodes) - - target_columns = {column for column, deg in column_graph.out_degree if deg == 0} - - result: Set[str] = set() - for column in target_columns: - # Let's drop all the count(*) and similard columns which are expression actually if it does not have an alias - if not any(ele in column.raw_name for ele in ["*", "(", ")"]): - result.add(str(column.raw_name)) - - # Reverting back all the previously renamed words which confuses the parser - result = {"date" if c == self._DATE_SWAP_TOKEN else c for c in result} - result = { - "timestamp" if c == self._TIMESTAMP_SWAP_TOKEN else c for c in list(result) - } - - # swap back renamed date column - return list(result) diff --git a/metadata-ingestion/src/datahub/utilities/sql_parser.py b/metadata-ingestion/src/datahub/utilities/sql_parser.py deleted file mode 100644 index b88f8fd8c73029..00000000000000 --- a/metadata-ingestion/src/datahub/utilities/sql_parser.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -import multiprocessing -import traceback -from multiprocessing import Process, Queue -from typing import Any, List, Optional, Tuple - -from datahub.utilities.sql_lineage_parser_impl import SqlLineageSQLParserImpl -from datahub.utilities.sql_parser_base import SQLParser - -logger = logging.getLogger(__name__) - - -def sql_lineage_parser_impl_func_wrapper( - queue: Optional[multiprocessing.Queue], sql_query: str, use_raw_names: bool = False -) -> Optional[Tuple[List[str], List[str], Any]]: - """ - The wrapper function that computes the tables and columns using the SqlLineageSQLParserImpl - and puts the results on the shared IPC queue. This is used to isolate SqlLineageSQLParserImpl - functionality in a separate process, and hence protect our sources from memory leaks originating in - the sqllineage module. - :param queue: The shared IPC queue on to which the results will be put. - :param sql_query: The SQL query to extract the tables & columns from. - :param use_raw_names: Parameter used to ignore sqllineage's default lowercasing. - :return: None. - """ - exception_details: Optional[Tuple[BaseException, str]] = None - tables: List[str] = [] - columns: List[str] = [] - try: - parser = SqlLineageSQLParserImpl(sql_query, use_raw_names) - tables = parser.get_tables() - columns = parser.get_columns() - except BaseException as e: - exc_msg = traceback.format_exc() - exception_details = (e, exc_msg) - logger.debug(exc_msg) - - if queue is not None: - queue.put((tables, columns, exception_details)) - return None - else: - return (tables, columns, exception_details) - - -class SqlLineageSQLParser(SQLParser): - def __init__( - self, - sql_query: str, - use_external_process: bool = False, - use_raw_names: bool = False, - ) -> None: - super().__init__(sql_query, use_external_process) - if use_external_process: - self.tables, self.columns = self._get_tables_columns_process_wrapped( - sql_query, use_raw_names - ) - else: - return_tuple = sql_lineage_parser_impl_func_wrapper( - None, sql_query, use_raw_names - ) - if return_tuple is not None: - ( - self.tables, - self.columns, - some_exception, - ) = return_tuple - - @staticmethod - def _get_tables_columns_process_wrapped( - sql_query: str, use_raw_names: bool = False - ) -> Tuple[List[str], List[str]]: - # Invoke sql_lineage_parser_impl_func_wrapper in a separate process to avoid - # memory leaks from sqllineage module used by SqlLineageSQLParserImpl. This will help - # shield our sources like lookml & redash, that need to parse a large number of SQL statements, - # from causing significant memory leaks in the datahub cli during ingestion. - queue: multiprocessing.Queue = Queue() - process: multiprocessing.Process = Process( - target=sql_lineage_parser_impl_func_wrapper, - args=(queue, sql_query, use_raw_names), - ) - process.start() - tables, columns, exception_details = queue.get(block=True) - if exception_details is not None: - raise exception_details[0](f"Sub-process exception: {exception_details[1]}") - return tables, columns - - def get_tables(self) -> List[str]: - return self.tables - - def get_columns(self) -> List[str]: - return self.columns - - -DefaultSQLParser = SqlLineageSQLParser diff --git a/metadata-ingestion/src/datahub/utilities/sql_parser_base.py b/metadata-ingestion/src/datahub/utilities/sql_parser_base.py deleted file mode 100644 index 8fd5dfaf4978d1..00000000000000 --- a/metadata-ingestion/src/datahub/utilities/sql_parser_base.py +++ /dev/null @@ -1,21 +0,0 @@ -from abc import ABCMeta, abstractmethod -from typing import List - - -class SqlParserException(Exception): - """Raised when sql parser fails""" - - pass - - -class SQLParser(metaclass=ABCMeta): - def __init__(self, sql_query: str, use_external_process: bool = True) -> None: - self._sql_query = sql_query - - @abstractmethod - def get_tables(self) -> List[str]: - pass - - @abstractmethod - def get_columns(self) -> List[str]: - pass diff --git a/metadata-ingestion/tests/unit/test_redash_source.py b/metadata-ingestion/tests/unit/test_redash_source.py index 2982fe76c4d4e7..32ab200847dc6c 100644 --- a/metadata-ingestion/tests/unit/test_redash_source.py +++ b/metadata-ingestion/tests/unit/test_redash_source.py @@ -710,9 +710,9 @@ def test_get_chart_snapshot_parse_table_names_from_sql(mocked_data_source): ), chartUrl="http://localhost:5000/queries/4#10", inputs=[ - "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.order_items,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.orders,PROD)", - "urn:li:dataset:(urn:li:dataPlatform:mysql,Rfam.staffs,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.order_items,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.orders,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mysql,rfam.staffs,PROD)", ], type="PIE", ) diff --git a/metadata-ingestion/tests/unit/utilities/test_utilities.py b/metadata-ingestion/tests/unit/utilities/test_utilities.py index 68da1bc1c01be2..91819bff41e629 100644 --- a/metadata-ingestion/tests/unit/utilities/test_utilities.py +++ b/metadata-ingestion/tests/unit/utilities/test_utilities.py @@ -1,8 +1,55 @@ import doctest +import re +from typing import List +from datahub.sql_parsing.schema_resolver import SchemaResolver +from datahub.sql_parsing.sqlglot_lineage import sqlglot_lineage from datahub.utilities.delayed_iter import delayed_iter from datahub.utilities.is_pytest import is_pytest_running -from datahub.utilities.sql_parser import SqlLineageSQLParser +from datahub.utilities.urns.dataset_urn import DatasetUrn + + +class SqlLineageSQLParser: + """ + It uses `sqlglot_lineage` to extract tables and columns, serving as a replacement for the `sqllineage` implementation, similar to BigQuery. + Reference: [BigQuery SQL Lineage Test](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/tests/unit/bigquery/test_bigquery_sql_lineage.py#L8). + """ + + _MYVIEW_SQL_TABLE_NAME_TOKEN = "__my_view__.__sql_table_name__" + _MYVIEW_LOOKER_TOKEN = "my_view.SQL_TABLE_NAME" + + def __init__(self, sql_query: str, platform: str = "bigquery") -> None: + # SqlLineageParser lowercarese tablenames and we need to replace Looker specific token which should be uppercased + sql_query = re.sub( + rf"(\${{{self._MYVIEW_LOOKER_TOKEN}}})", + rf"{self._MYVIEW_SQL_TABLE_NAME_TOKEN}", + sql_query, + ) + self.sql_query = sql_query + self.schema_resolver = SchemaResolver(platform=platform) + self.result = sqlglot_lineage(sql_query, self.schema_resolver) + + def get_tables(self) -> List[str]: + ans = [] + for urn in self.result.in_tables: + table_ref = DatasetUrn.from_string(urn) + ans.append(str(table_ref.name)) + + result = [ + self._MYVIEW_LOOKER_TOKEN if c == self._MYVIEW_SQL_TABLE_NAME_TOKEN else c + for c in ans + ] + # Sort tables to make the list deterministic + result.sort() + + return result + + def get_columns(self) -> List[str]: + ans = [] + for col_info in self.result.column_lineage or []: + for col_ref in col_info.upstreams: + ans.append(col_ref.column) + return ans def test_delayed_iter(): @@ -121,7 +168,7 @@ def test_sqllineage_sql_parser_get_columns_with_alias_and_count_star(): columns_list = SqlLineageSQLParser(sql_query).get_columns() columns_list.sort() - assert columns_list == ["a", "b", "count", "test"] + assert columns_list == ["a", "b", "c"] def test_sqllineage_sql_parser_get_columns_with_more_complex_join(): @@ -145,7 +192,7 @@ def test_sqllineage_sql_parser_get_columns_with_more_complex_join(): columns_list = SqlLineageSQLParser(sql_query).get_columns() columns_list.sort() - assert columns_list == ["bs", "pi", "pt", "pu", "v"] + assert columns_list == ["bs", "pi", "tt", "tt", "v"] def test_sqllineage_sql_parser_get_columns_complex_query_with_union(): @@ -198,7 +245,7 @@ def test_sqllineage_sql_parser_get_columns_complex_query_with_union(): columns_list = SqlLineageSQLParser(sql_query).get_columns() columns_list.sort() - assert columns_list == ["c", "date", "e", "u", "x"] + assert columns_list == ["c", "c", "e", "e", "e", "e", "u", "u", "x", "x"] def test_sqllineage_sql_parser_get_tables_from_templated_query(): @@ -239,7 +286,7 @@ def test_sqllineage_sql_parser_with_weird_lookml_query(): """ columns_list = SqlLineageSQLParser(sql_query).get_columns() columns_list.sort() - assert columns_list == ["aliased_platform", "country", "date"] + assert columns_list == [] def test_sqllineage_sql_parser_tables_from_redash_query(): @@ -276,13 +323,7 @@ def test_sqllineage_sql_parser_tables_with_special_names(): "hour-table", "timestamp-table", ] - expected_columns = [ - "column-admin", - "column-data", - "column-date", - "column-hour", - "column-timestamp", - ] + expected_columns: List[str] = [] assert sorted(SqlLineageSQLParser(sql_query).get_tables()) == expected_tables assert sorted(SqlLineageSQLParser(sql_query).get_columns()) == expected_columns From 1f389c1d36545e45ddf5ab6d4d243dafaa0d6029 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:00:58 -0600 Subject: [PATCH 162/174] fix(entity-service): prevent mutation of systemMetdata on prev (#12081) --- .../metadata/entity/EntityServiceImpl.java | 30 ++++++++++++------- .../metadata/entity/EntityServiceTest.java | 5 ++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java index 059a6b7ed0aea3..d14990f93d22d9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/entity/EntityServiceImpl.java @@ -2536,16 +2536,12 @@ private UpdateAspectResult ingestAspectToLocalDB( Optional latestSystemMetadataDiff = systemMetadataDiff( txContext, + writeItem.getUrn(), previousBatchAspect.getSystemMetadata(), writeItem.getSystemMetadata(), databaseAspect == null ? null : databaseAspect.getSystemMetadata()); if (latestSystemMetadataDiff.isPresent()) { - // Update previous version since that is what is re-written - previousBatchAspect - .getEntityAspect() - .setSystemMetadata(RecordUtils.toJsonString(latestSystemMetadataDiff.get())); - // Inserts & update order is not guaranteed, flush the insert for potential updates within // same tx if (databaseAspect == null && txContext != null) { @@ -2560,13 +2556,25 @@ private UpdateAspectResult ingestAspectToLocalDB( conditionalLogLevel( txContext, String.format( - "Update aspect with name %s, urn %s, txContext: %s, databaseAspect: %s, newAspect: %s", + "Update aspect with name %s, urn %s, txContext: %s, databaseAspect: %s, newMetadata: %s newSystemMetadata: %s", previousBatchAspect.getAspectName(), previousBatchAspect.getUrn(), txContext != null, databaseAspect == null ? null : databaseAspect.getEntityAspect(), - previousBatchAspect.getEntityAspect())); - aspectDao.saveAspect(txContext, previousBatchAspect.getEntityAspect(), false); + previousBatchAspect.getEntityAspect().getMetadata(), + latestSystemMetadataDiff.get())); + + aspectDao.saveAspect( + txContext, + previousBatchAspect.getUrnRaw(), + previousBatchAspect.getAspectName(), + previousBatchAspect.getMetadataRaw(), + previousBatchAspect.getCreatedBy(), + null, + previousBatchAspect.getCreatedOn(), + RecordUtils.toJsonString(latestSystemMetadataDiff.get()), + previousBatchAspect.getVersion(), + false); // metrics aspectDao.incrementWriteMetrics( @@ -2661,13 +2669,14 @@ private static boolean shouldAspectEmitChangeLog(@Nonnull final AspectSpec aspec private static Optional systemMetadataDiff( @Nullable TransactionContext txContext, + @Nonnull Urn urn, @Nullable SystemMetadata previous, @Nonnull SystemMetadata current, @Nullable SystemMetadata database) { SystemMetadata latestSystemMetadata = GenericRecordUtils.copy(previous, SystemMetadata.class); - latestSystemMetadata.setLastRunId(previous.getRunId(), SetMode.REMOVE_IF_NULL); + latestSystemMetadata.setLastRunId(latestSystemMetadata.getRunId(), SetMode.REMOVE_IF_NULL); latestSystemMetadata.setLastObserved(current.getLastObserved(), SetMode.IGNORE_NULL); latestSystemMetadata.setRunId(current.getRunId(), SetMode.REMOVE_IF_NULL); @@ -2677,7 +2686,8 @@ private static Optional systemMetadataDiff( conditionalLogLevel( txContext, String.format( - "systemMetdataDiff: %s != %s AND %s", + "systemMetdataDiff urn %s, %s != %s AND %s", + urn, RecordUtils.toJsonString(latestSystemMetadata), previous == null ? null : RecordUtils.toJsonString(previous), database == null ? null : RecordUtils.toJsonString(database))); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java index 18d277cacbbe26..4c42815a80f3f1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/entity/EntityServiceTest.java @@ -627,7 +627,7 @@ public void testReingestLineageAspect() throws Exception { restateChangeLog.setSystemMetadata(futureSystemMetadata); restateChangeLog.setPreviousAspectValue(aspect); restateChangeLog.setPreviousSystemMetadata( - simulatePullFromDB(futureSystemMetadata, SystemMetadata.class)); + simulatePullFromDB(initialSystemMetadata, SystemMetadata.class)); restateChangeLog.setEntityKeyAspect( GenericRecordUtils.serializeAspect( EntityKeyUtils.convertUrnToEntityKey( @@ -705,8 +705,7 @@ public void testReingestLineageProposal() throws Exception { restateChangeLog.setAspect(genericAspect); restateChangeLog.setSystemMetadata(futureSystemMetadata); restateChangeLog.setPreviousAspectValue(genericAspect); - restateChangeLog.setPreviousSystemMetadata( - simulatePullFromDB(futureSystemMetadata, SystemMetadata.class)); + restateChangeLog.setPreviousSystemMetadata(simulatePullFromDB(metadata1, SystemMetadata.class)); Map latestAspects = _entityServiceImpl.getLatestAspectsForUrn( From a68836e14c5b6a7dec42ff130c116a92d62127f2 Mon Sep 17 00:00:00 2001 From: Chakru <161002324+chakru-r@users.noreply.github.com> Date: Tue, 10 Dec 2024 23:24:43 +0530 Subject: [PATCH 163/174] build(datahub-frontend): enable code-coverage (#12072) --- datahub-frontend/build.gradle | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle index 7750e169b11fbe..5cc5af50d217ba 100644 --- a/datahub-frontend/build.gradle +++ b/datahub-frontend/build.gradle @@ -4,8 +4,9 @@ plugins { id 'org.gradle.playframework' } -apply from: "../gradle/versioning/versioning.gradle" +apply from: '../gradle/versioning/versioning.gradle' apply from: './play.gradle' +apply from: '../gradle/coverage/java-coverage.gradle' ext { docker_repo = 'datahub-frontend-react' @@ -18,6 +19,13 @@ java { } } +test { + jacoco { + // jacoco instrumentation is failing when dealing with code of this dependency, excluding it. + excludes = ["com/gargoylesoftware/**"] + } +} + model { // Must specify the dependency here as "stage" is added by rule based model. tasks.myTar { From bcf230f87ffd1cf090d84bba3a394a7ac771a13a Mon Sep 17 00:00:00 2001 From: Chakru <161002324+chakru-r@users.noreply.github.com> Date: Wed, 11 Dec 2024 00:47:30 +0530 Subject: [PATCH 164/174] build(ci): codecov integration (#12073) --- .github/workflows/airflow-plugin.yml | 4 ++-- .github/workflows/build-and-test.yml | 10 ++++++++++ .github/workflows/dagster-plugin.yml | 4 ++-- .github/workflows/gx-plugin.yml | 4 ++-- .github/workflows/metadata-ingestion.yml | 4 ++-- .github/workflows/metadata-io.yml | 9 +++++++++ .github/workflows/prefect-plugin.yml | 4 ++-- 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml index 1fdfc52857b011..eefa02be4f1af8 100644 --- a/.github/workflows/airflow-plugin.yml +++ b/.github/workflows/airflow-plugin.yml @@ -80,10 +80,10 @@ jobs: !**/binary/** - name: Upload coverage to Codecov if: always() - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - directory: . + directory: ./build/coverage-reports/ fail_ci_if_error: false flags: airflow,airflow-${{ matrix.extra_pip_extras }} name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a5889b2d2f92de..1b10fe6e74372b 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -126,6 +126,16 @@ jobs: !**/binary/** - name: Ensure codegen is updated uses: ./.github/actions/ensure-codegen-updated + - name: Upload coverage to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + directory: ./build/coverage-reports/ + fail_ci_if_error: false + flags: ${{ matrix.timezone }} + name: ${{ matrix.command }} + verbose: true quickstart-compose-validation: runs-on: ubuntu-latest diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml index 37b6c93ec841ab..f512dcf8f3ffd4 100644 --- a/.github/workflows/dagster-plugin.yml +++ b/.github/workflows/dagster-plugin.yml @@ -66,10 +66,10 @@ jobs: **/junit.*.xml - name: Upload coverage to Codecov if: always() - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - directory: . + directory: ./build/coverage-reports/ fail_ci_if_error: false flags: dagster-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }} name: pytest-dagster diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml index aa7c3f069c7654..595438bd6e4a90 100644 --- a/.github/workflows/gx-plugin.yml +++ b/.github/workflows/gx-plugin.yml @@ -70,10 +70,10 @@ jobs: **/junit.*.xml - name: Upload coverage to Codecov if: always() - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - directory: . + directory: ./build/coverage-reports/ fail_ci_if_error: false flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }} name: pytest-gx diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index c0eafe891fb0aa..49def2a863c565 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -94,10 +94,10 @@ jobs: !**/binary/** - name: Upload coverage to Codecov if: ${{ always() && matrix.python-version == '3.10' }} - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - directory: . + directory: ./build/coverage-reports/ fail_ci_if_error: false flags: pytest-${{ matrix.command }} name: pytest-${{ matrix.command }} diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index 5ee2223d71b039..2225baecde64c6 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -81,6 +81,15 @@ jobs: !**/binary/** - name: Ensure codegen is updated uses: ./.github/actions/ensure-codegen-updated + - name: Upload coverage to Codecov + if: ${{ always()}} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + directory: ./build/coverage-reports/ + fail_ci_if_error: false + name: metadata-io-test + verbose: true event-file: runs-on: ubuntu-latest diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml index b0af00f92b7727..3c75e8fe9a62ff 100644 --- a/.github/workflows/prefect-plugin.yml +++ b/.github/workflows/prefect-plugin.yml @@ -67,10 +67,10 @@ jobs: !**/binary/** - name: Upload coverage to Codecov if: always() - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} - directory: . + directory: ./build/coverage-reports/ fail_ci_if_error: false flags: prefect,prefect-${{ matrix.extra_pip_extras }} name: pytest-prefect-${{ matrix.python-version }} From a290b24a7cdd67f2c53ad90e60b230043ee1ea2a Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 10 Dec 2024 13:25:41 -0600 Subject: [PATCH 165/174] fix(openapi): adds in previously ignored keep alive value (#12068) --- docs/how/updating-datahub.md | 1 + .../openapi/controller/GenericEntitiesController.java | 6 ++++-- .../io/datahubproject/openapi/v3/OpenAPIV3Generator.java | 7 +++++++ .../openapi/v3/controller/EntityController.java | 4 +++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index d8fe06abad6252..073f68db230634 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -39,6 +39,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information. - #12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2. - #12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation. +- OpenAPI Update: PIT Keep Alive parameter added to scroll. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set. ### Breaking Changes diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index c17a4a6294f015..425646d4282149 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -199,7 +199,9 @@ public ResponseEntity getEntities( @RequestParam(value = "skipCache", required = false, defaultValue = "false") Boolean skipCache, @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false") - Boolean includeSoftDelete) + Boolean includeSoftDelete, + @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m") + String pitKeepALive) throws URISyntaxException { EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); @@ -241,7 +243,7 @@ public ResponseEntity getEntities( null, sortCriteria, scrollId, - null, + pitKeepALive, count); if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) { diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index d179ea8f3a0682..68ed316573f77a 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -45,6 +45,7 @@ public class OpenAPIV3Generator { private static final String NAME_VERSION = "version"; private static final String NAME_SCROLL_ID = "scrollId"; private static final String NAME_INCLUDE_SOFT_DELETE = "includeSoftDelete"; + private static final String NAME_PIT_KEEP_ALIVE = "pitKeepAlive"; private static final String PROPERTY_VALUE = "value"; private static final String PROPERTY_URN = "urn"; private static final String PROPERTY_PATCH = "patch"; @@ -502,6 +503,12 @@ private static PathItem buildGenericListEntitiesPath() { .name(NAME_SKIP_CACHE) .description("Skip cache when listing entities.") .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_PIT_KEEP_ALIVE) + .description( + "Point In Time keep alive, accepts a time based string like \"5m\" for five minutes.") + .schema(new Schema().type(TYPE_STRING)._default("5m")), new Parameter().$ref("#/components/parameters/PaginationCount" + MODEL_VERSION), new Parameter().$ref("#/components/parameters/ScrollId" + MODEL_VERSION), new Parameter().$ref("#/components/parameters/SortBy" + MODEL_VERSION), diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index aa659b196f1872..5544fb845b2687 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -146,6 +146,8 @@ public ResponseEntity scrollEntities( Boolean skipCache, @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false") Boolean includeSoftDelete, + @RequestParam(value = "pitKeepAlive", required = false, defaultValue = "5m") + String pitKeepALive, @RequestBody @Nonnull GenericEntityAspectsBodyV3 entityAspectsBody) throws URISyntaxException { @@ -202,7 +204,7 @@ public ResponseEntity scrollEntities( null, sortCriteria, scrollId, - null, + pitKeepALive, count); if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) { From 84e50d8e76a1c44be39a0600d426b58e2b8c690b Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Tue, 10 Dec 2024 15:06:57 -0500 Subject: [PATCH 166/174] feat(ui) Add alchemy component library to FE (#12054) --- datahub-web-react/.storybook/DocTemplate.mdx | 42 + datahub-web-react/.storybook/main.js | 25 + .../.storybook/manager-head.html | 33 + datahub-web-react/.storybook/manager.js | 15 + .../.storybook/preview-head.html | 33 + datahub-web-react/.storybook/preview.js | 84 + .../.storybook/storybook-logo.svg | 1 + .../.storybook/storybook-theme.css | 263 ++++ .../.storybook/storybook-theme.js | 47 + .../.storybook/styledComponents.ts | 36 + .../.storybook/webpack.config.js | 13 + datahub-web-react/package.json | 20 +- .../alchemy-components/.docs/Contributing.mdx | 43 + .../alchemy-components/.docs/DesignTokens.mdx | 63 + .../src/alchemy-components/.docs/Icons.mdx | 34 + .../src/alchemy-components/.docs/Intro.mdx | 14 + .../alchemy-components/.docs/StyleGuide.mdx | 209 +++ .../.docs/mdx-components/CodeBlock.tsx | 24 + .../.docs/mdx-components/CopyButton.tsx | 16 + .../.docs/mdx-components/GridList.tsx | 32 + .../mdx-components/IconGalleryWithSearch.tsx | 291 ++++ .../.docs/mdx-components/components.ts | 110 ++ .../.docs/mdx-components/index.ts | 6 + .../.docs/mdx-components/utils.ts | 15 + .../src/alchemy-components/README.mdx | 73 + .../components/Avatar/Avatar.stories.tsx | 133 ++ .../components/Avatar/Avatar.tsx | 40 + .../Avatar/_tests_/getNameInitials.test.ts | 34 + .../components/Avatar/components.ts | 51 + .../components/Avatar/index.ts | 1 + .../components/Avatar/types.ts | 10 + .../components/Avatar/utils.ts | 64 + .../components/Badge/Badge.stories.tsx | 102 ++ .../components/Badge/Badge.tsx | 29 + .../components/Badge/components.ts | 6 + .../components/Badge/index.ts | 1 + .../components/Badge/types.ts | 8 + .../components/Badge/utils.ts | 15 + .../components/BarChart/BarChart.stories.tsx | 90 ++ .../components/BarChart/BarChart.tsx | 152 ++ .../components/BarChart/components.tsx | 34 + .../components/BarChart/index.ts | 1 + .../components/BarChart/types.ts | 18 + .../components/BarChart/utils.ts | 26 + .../components/Button/Button.stories.tsx | 203 +++ .../components/Button/Button.tsx | 60 + .../components/Button/components.ts | 27 + .../components/Button/index.ts | 2 + .../components/Button/types.ts | 16 + .../components/Button/utils.ts | 238 +++ .../components/Card/Card.stories.tsx | 141 ++ .../components/Card/Card.tsx | 48 + .../components/Card/components.ts | 59 + .../components/Card/index.ts | 2 + .../components/Card/types.ts | 13 + .../components/Checkbox/Checkbox.stories.tsx | 156 ++ .../components/Checkbox/Checkbox.tsx | 103 ++ .../components/Checkbox/components.ts | 91 ++ .../components/Checkbox/index.ts | 2 + .../components/Checkbox/types.ts | 16 + .../components/Checkbox/utils.ts | 27 + .../components/Heading/Heading.stories.tsx | 98 ++ .../components/Heading/Heading.tsx | 38 + .../components/Heading/components.ts | 70 + .../components/Heading/index.ts | 2 + .../components/Heading/types.ts | 9 + .../components/Icon/Icon.stories.tsx | 131 ++ .../components/Icon/Icon.tsx | 59 + .../components/Icon/components.ts | 19 + .../components/Icon/constants.ts | 547 +++++++ .../components/Icon/index.ts | 3 + .../components/Icon/types.ts | 23 + .../components/Icon/utils.ts | 29 + .../components/Input/Input.stories.tsx | 177 +++ .../components/Input/Input.tsx | 97 ++ .../components/Input/components.ts | 92 ++ .../components/Input/index.ts | 2 + .../components/Input/types.ts | 22 + .../components/Input/utils.ts | 5 + .../LineChart/LineChart.stories.tsx | 96 ++ .../components/LineChart/LineChart.tsx | 178 +++ .../components/LineChart/components.tsx | 8 + .../components/LineChart/index.ts | 1 + .../components/LineChart/types.ts | 22 + .../PageTitle/PageTitle.stories.tsx | 71 + .../components/PageTitle/PageTitle.tsx | 17 + .../components/PageTitle/components.ts | 52 + .../components/PageTitle/index.ts | 1 + .../components/PageTitle/types.ts | 8 + .../components/PageTitle/utils.ts | 27 + .../components/Pills/Pill.stories.tsx | 126 ++ .../components/Pills/Pill.tsx | 42 + .../components/Pills/components.ts | 33 + .../components/Pills/index.ts | 1 + .../components/Pills/types.ts | 18 + .../components/Pills/utils.ts | 147 ++ .../components/Popover/Popover.tsx | 6 + .../components/Popover/index.ts | 1 + .../components/Radio/Radio.stories.tsx | 136 ++ .../components/Radio/Radio.tsx | 89 ++ .../components/Radio/components.ts | 83 + .../components/Radio/types.ts | 16 + .../components/Radio/utils.ts | 27 + .../components/Select/BasicSelect.tsx | 339 ++++ .../components/Select/Nested/NestedOption.tsx | 309 ++++ .../components/Select/Nested/NestedSelect.tsx | 312 ++++ .../components/Select/Nested/types.ts | 9 + .../components/Select/Select.stories.tsx | 431 +++++ .../components/Select/Select.tsx | 65 + .../components/Select/SimpleSelect.tsx | 299 ++++ .../components/Select/components.ts | 235 +++ .../components/Select/index.ts | 3 + .../components/Select/types.ts | 61 + .../components/Select/utils.ts | 125 ++ .../components/Switch/Switch.stories.tsx | 169 ++ .../components/Switch/Switch.tsx | 74 + .../components/Switch/components.ts | 118 ++ .../components/Switch/index.ts | 2 + .../components/Switch/types.ts | 21 + .../components/Switch/utils.ts | 97 ++ .../components/Table/Table.stories.tsx | 162 ++ .../components/Table/Table.tsx | 115 ++ .../components/Table/components.ts | 94 ++ .../components/Table/index.ts | 2 + .../components/Table/types.ts | 21 + .../components/Table/utils.ts | 73 + .../components/Text/Text.stories.tsx | 100 ++ .../components/Text/Text.tsx | 33 + .../components/Text/components.ts | 50 + .../components/Text/index.ts | 2 + .../components/Text/types.ts | 9 + .../components/TextArea/TextArea.stories.tsx | 159 ++ .../components/TextArea/TextArea.tsx | 80 + .../components/TextArea/components.ts | 106 ++ .../components/TextArea/index.ts | 2 + .../components/TextArea/types.ts | 15 + .../components/Tooltip/Tooltip.tsx | 6 + .../components/Tooltip/index.ts | 1 + .../components/commonStyles.ts | 23 + .../components/dataviz/utils.ts | 11 + .../src/alchemy-components/index.ts | 23 + .../theme/config/constants.ts | 1 + .../alchemy-components/theme/config/index.ts | 2 + .../alchemy-components/theme/config/types.ts | 47 + .../theme/foundations/blur.ts | 12 + .../theme/foundations/borders.ts | 9 + .../theme/foundations/breakpoints.ts | 10 + .../theme/foundations/colors.ts | 98 ++ .../theme/foundations/index.ts | 27 + .../theme/foundations/radius.ts | 9 + .../theme/foundations/shadows.ts | 16 + .../theme/foundations/sizes.ts | 7 + .../theme/foundations/spacing.ts | 12 + .../theme/foundations/transform.ts | 10 + .../theme/foundations/transition.ts | 32 + .../theme/foundations/typography.ts | 52 + .../theme/foundations/zIndex.ts | 17 + .../src/alchemy-components/theme/index.ts | 30 + .../theme/semantic-tokens.ts | 21 + .../src/alchemy-components/theme/utils.ts | 62 + datahub-web-react/src/fonts/Mulish-Black.ttf | Bin 0 -> 106576 bytes .../src/fonts/Mulish-BlackItalic.ttf | Bin 0 -> 110200 bytes datahub-web-react/src/fonts/Mulish-Bold.ttf | Bin 0 -> 106576 bytes .../src/fonts/Mulish-BoldItalic.ttf | Bin 0 -> 110260 bytes .../src/fonts/Mulish-ExtraBold.ttf | Bin 0 -> 106536 bytes .../src/fonts/Mulish-ExtraBoldItalic.ttf | Bin 0 -> 110256 bytes .../src/fonts/Mulish-ExtraLight.ttf | Bin 0 -> 106380 bytes .../src/fonts/Mulish-ExtraLightItalic.ttf | Bin 0 -> 110000 bytes datahub-web-react/src/fonts/Mulish-Italic.ttf | Bin 0 -> 110060 bytes datahub-web-react/src/fonts/Mulish-Light.ttf | Bin 0 -> 106572 bytes .../src/fonts/Mulish-LightItalic.ttf | Bin 0 -> 110192 bytes datahub-web-react/src/fonts/Mulish-Medium.ttf | Bin 0 -> 106568 bytes .../src/fonts/Mulish-MediumItalic.ttf | Bin 0 -> 110160 bytes .../src/fonts/Mulish-Regular.ttf | Bin 0 -> 106528 bytes .../src/fonts/Mulish-SemiBold.ttf | Bin 0 -> 106508 bytes .../src/fonts/Mulish-SemiBoldItalic.ttf | Bin 0 -> 110120 bytes datahub-web-react/tsconfig.json | 30 +- datahub-web-react/vite.config.ts | 28 + datahub-web-react/yarn.lock | 1384 ++++++++++++++++- 179 files changed, 11848 insertions(+), 14 deletions(-) create mode 100644 datahub-web-react/.storybook/DocTemplate.mdx create mode 100644 datahub-web-react/.storybook/main.js create mode 100644 datahub-web-react/.storybook/manager-head.html create mode 100644 datahub-web-react/.storybook/manager.js create mode 100644 datahub-web-react/.storybook/preview-head.html create mode 100644 datahub-web-react/.storybook/preview.js create mode 100644 datahub-web-react/.storybook/storybook-logo.svg create mode 100644 datahub-web-react/.storybook/storybook-theme.css create mode 100644 datahub-web-react/.storybook/storybook-theme.js create mode 100644 datahub-web-react/.storybook/styledComponents.ts create mode 100644 datahub-web-react/.storybook/webpack.config.js create mode 100644 datahub-web-react/src/alchemy-components/.docs/Contributing.mdx create mode 100644 datahub-web-react/src/alchemy-components/.docs/DesignTokens.mdx create mode 100644 datahub-web-react/src/alchemy-components/.docs/Icons.mdx create mode 100644 datahub-web-react/src/alchemy-components/.docs/Intro.mdx create mode 100644 datahub-web-react/src/alchemy-components/.docs/StyleGuide.mdx create mode 100644 datahub-web-react/src/alchemy-components/.docs/mdx-components/CodeBlock.tsx create mode 100644 datahub-web-react/src/alchemy-components/.docs/mdx-components/CopyButton.tsx create mode 100644 datahub-web-react/src/alchemy-components/.docs/mdx-components/GridList.tsx create mode 100644 datahub-web-react/src/alchemy-components/.docs/mdx-components/IconGalleryWithSearch.tsx create mode 100644 datahub-web-react/src/alchemy-components/.docs/mdx-components/components.ts create mode 100644 datahub-web-react/src/alchemy-components/.docs/mdx-components/index.ts create mode 100644 datahub-web-react/src/alchemy-components/.docs/mdx-components/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/README.mdx create mode 100644 datahub-web-react/src/alchemy-components/components/Avatar/Avatar.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Avatar/Avatar.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Avatar/_tests_/getNameInitials.test.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Avatar/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Avatar/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Avatar/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Avatar/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Badge/Badge.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Badge/Badge.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Badge/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Badge/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Badge/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Badge/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/BarChart/BarChart.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/BarChart/BarChart.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/BarChart/components.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/BarChart/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/BarChart/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/BarChart/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Button/Button.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Button/Button.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Button/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Button/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Button/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Button/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Card/Card.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Card/Card.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Card/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Card/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Card/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Checkbox/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Checkbox/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Checkbox/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Checkbox/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Heading/Heading.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Heading/Heading.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Heading/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Heading/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Heading/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Icon/Icon.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Icon/Icon.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Icon/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Icon/constants.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Icon/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Icon/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Icon/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Input/Input.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Input/Input.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Input/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Input/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Input/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Input/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/LineChart/LineChart.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/LineChart/LineChart.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/LineChart/components.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/LineChart/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/LineChart/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/PageTitle/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/PageTitle/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/PageTitle/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/PageTitle/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Pills/Pill.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Pills/Pill.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Pills/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Pills/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Pills/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Pills/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Popover/Popover.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Popover/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Radio/Radio.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Radio/Radio.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Radio/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Radio/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Radio/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Select/BasicSelect.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Select/Nested/NestedOption.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Select/Nested/NestedSelect.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Select/Nested/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Select/Select.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Select/Select.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Select/SimpleSelect.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Select/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Select/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Select/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Select/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Switch/Switch.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Switch/Switch.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Switch/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Switch/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Switch/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Switch/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Table/Table.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Table/Table.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Table/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Table/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Table/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Table/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Text/Text.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Text/Text.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Text/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Text/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Text/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/TextArea/TextArea.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/TextArea/TextArea.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/TextArea/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/TextArea/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/TextArea/types.ts create mode 100644 datahub-web-react/src/alchemy-components/components/Tooltip/Tooltip.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/Tooltip/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/commonStyles.ts create mode 100644 datahub-web-react/src/alchemy-components/components/dataviz/utils.ts create mode 100644 datahub-web-react/src/alchemy-components/index.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/config/constants.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/config/index.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/config/types.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/blur.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/borders.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/breakpoints.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/colors.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/index.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/radius.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/shadows.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/sizes.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/spacing.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/transform.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/transition.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/typography.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/foundations/zIndex.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/index.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/semantic-tokens.ts create mode 100644 datahub-web-react/src/alchemy-components/theme/utils.ts create mode 100644 datahub-web-react/src/fonts/Mulish-Black.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-BlackItalic.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-Bold.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-BoldItalic.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-ExtraBold.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-ExtraBoldItalic.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-ExtraLight.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-ExtraLightItalic.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-Italic.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-Light.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-LightItalic.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-Medium.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-MediumItalic.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-Regular.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-SemiBold.ttf create mode 100644 datahub-web-react/src/fonts/Mulish-SemiBoldItalic.ttf diff --git a/datahub-web-react/.storybook/DocTemplate.mdx b/datahub-web-react/.storybook/DocTemplate.mdx new file mode 100644 index 00000000000000..9ea1250075e11f --- /dev/null +++ b/datahub-web-react/.storybook/DocTemplate.mdx @@ -0,0 +1,42 @@ +import React from 'react'; + +import { ThemeProvider } from 'styled-components'; +import { GlobalStyle } from './styledComponents'; + +import { Meta, Title, Subtitle, Description, Primary, Controls, Stories } from '@storybook/blocks'; +import { CodeBlock } from '../src/alchemy-components/.docs/mdx-components'; + +{/* + * 👇 The isTemplate property is required to tell Storybook that this is a template + * See https://storybook.js.org/docs/api/doc-block-meta + * to learn how to use +*/} + + + + + + + + + <Subtitle /> + + <div className="docsDescription"> + <Description /> + </div> + + <br /> + + ### Import + + <CodeBlock /> + + <br/> + + ### Customize + + <Primary /> + <Controls /> + + <Stories /> +</ThemeProvider> \ No newline at end of file diff --git a/datahub-web-react/.storybook/main.js b/datahub-web-react/.storybook/main.js new file mode 100644 index 00000000000000..2b92dffd88eb3a --- /dev/null +++ b/datahub-web-react/.storybook/main.js @@ -0,0 +1,25 @@ +// Docs for badges: https://storybook.js.org/addons/@geometricpanda/storybook-addon-badges + +export default { + framework: '@storybook/react-vite', + features: { + buildStoriesJson: true, + }, + core: { + disableTelemetry: true, + }, + stories: [ + '../src/alchemy-components/.docs/*.mdx', + '../src/alchemy-components/components/**/*.stories.@(js|jsx|mjs|ts|tsx)' + ], + addons: [ + '@storybook/addon-onboarding', + '@storybook/addon-essentials', + '@storybook/addon-interactions', + '@storybook/addon-links', + '@geometricpanda/storybook-addon-badges', + ], + typescript: { + reactDocgen: 'react-docgen-typescript', + }, +} \ No newline at end of file diff --git a/datahub-web-react/.storybook/manager-head.html b/datahub-web-react/.storybook/manager-head.html new file mode 100644 index 00000000000000..98e6a2895f45c7 --- /dev/null +++ b/datahub-web-react/.storybook/manager-head.html @@ -0,0 +1,33 @@ +<style type="text/css"> + /* Regular */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 400; + src: url('../src/fonts/Mulish-Regular.ttf') format('truetype'); + } + + /* Medium */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 500; + src: url('../src/fonts/Mulish-Medium.ttf') format('truetype'); + } + + /* SemiBold */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 600; + src: url('../src/fonts/Mulish-SemiBold.ttf') format('truetype'); + } + + /* Bold */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 700; + src: url('../src/fonts/Mulish-Bold.ttf') format('truetype'); + } +</style> \ No newline at end of file diff --git a/datahub-web-react/.storybook/manager.js b/datahub-web-react/.storybook/manager.js new file mode 100644 index 00000000000000..6e9c62dd96c23f --- /dev/null +++ b/datahub-web-react/.storybook/manager.js @@ -0,0 +1,15 @@ +import './storybook-theme.css'; + +import { addons } from '@storybook/manager-api'; +import acrylTheme from './storybook-theme.js'; + +// Theme setup +addons.setConfig({ + theme: acrylTheme, +}); + +// Favicon +const link = document.createElement('link'); +link.setAttribute('rel', 'shortcut icon'); +link.setAttribute('href', 'https://www.acryldata.io/icons/favicon.ico'); +document.head.appendChild(link); \ No newline at end of file diff --git a/datahub-web-react/.storybook/preview-head.html b/datahub-web-react/.storybook/preview-head.html new file mode 100644 index 00000000000000..98e6a2895f45c7 --- /dev/null +++ b/datahub-web-react/.storybook/preview-head.html @@ -0,0 +1,33 @@ +<style type="text/css"> + /* Regular */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 400; + src: url('../src/fonts/Mulish-Regular.ttf') format('truetype'); + } + + /* Medium */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 500; + src: url('../src/fonts/Mulish-Medium.ttf') format('truetype'); + } + + /* SemiBold */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 600; + src: url('../src/fonts/Mulish-SemiBold.ttf') format('truetype'); + } + + /* Bold */ + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 700; + src: url('../src/fonts/Mulish-Bold.ttf') format('truetype'); + } +</style> \ No newline at end of file diff --git a/datahub-web-react/.storybook/preview.js b/datahub-web-react/.storybook/preview.js new file mode 100644 index 00000000000000..a497ce7bccf3c8 --- /dev/null +++ b/datahub-web-react/.storybook/preview.js @@ -0,0 +1,84 @@ +import './storybook-theme.css'; +// FYI: import of antd styles required to show components based on it correctly +import 'antd/dist/antd.css'; + +import { BADGE, defaultBadgesConfig } from '@geometricpanda/storybook-addon-badges'; +import DocTemplate from './DocTemplate.mdx'; + +const preview = { + tags: ['!dev', 'autodocs'], + parameters: { + previewTabs: { + 'storybook/docs/panel': { index: -1 }, + }, + controls: { + matchers: { + color: /(background|color)$/i, + date: /Date$/i, + }, + }, + options: { + storySort: { + method: 'alphabetical', + order: [ + // Order of Docs Pages + 'Introduction', + 'Style Guide', + 'Design Tokens', + 'Style Utilities', + 'Icons', + + // Order of Components + 'Layout', + 'Forms', + 'Data Display', + 'Feedback', + 'Typography', + 'Overlay', + 'Disclosure', + 'Navigation', + 'Media', + 'Other', + ], + locales: '', + }, + }, + docs: { + page: DocTemplate, + toc: { + disable: false, + }, + docs: { + source: { + format: true, + }, + }, + }, + + // Reconfig the premade badges with better titles + badgesConfig: { + stable: { + ...defaultBadgesConfig[BADGE.STABLE], + title: 'Stable', + tooltip: 'This component is stable but may have frequent changes. Use at own discretion.', + }, + productionReady: { + ...defaultBadgesConfig[BADGE.STABLE], + title: 'Production Ready', + tooltip: 'This component is production ready and has been tested in a production environment.', + }, + WIP: { + ...defaultBadgesConfig[BADGE.BETA], + title: 'WIP', + tooltip: 'This component is a work in progress and may not be fully functional or tested.', + }, + readyForDesignReview: { + ...defaultBadgesConfig[BADGE.NEEDS_REVISION], + title: 'Ready for Design Review', + tooltip: 'This component is ready for design review and feedback.', + }, + }, + }, +}; + +export default preview; diff --git a/datahub-web-react/.storybook/storybook-logo.svg b/datahub-web-react/.storybook/storybook-logo.svg new file mode 100644 index 00000000000000..5cc86813b59336 --- /dev/null +++ b/datahub-web-react/.storybook/storybook-logo.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 116.71 125.19"><defs><style>.cls-1{fill:#08303a;}.cls-2{fill:#11696b;}.cls-3{fill:#20d3bd;}</style></defs><g id="artwork"><path class="cls-1" d="M96.39,34.23,79.87,11.08a26.43,26.43,0,0,0-43,0L20.32,34.23A26.42,26.42,0,0,0,41.83,76h33A26.42,26.42,0,0,0,96.39,34.23ZM74.87,68h-33a18.42,18.42,0,0,1-15-29.12L43.35,15.72a18.43,18.43,0,0,1,30,0L89.87,38.88A18.42,18.42,0,0,1,74.87,68Z"/><path class="cls-2" d="M105.89,72.32,73,26.24a18,18,0,0,0-29.31,0L10.82,72.32a18,18,0,0,0,14.65,28.46H91.24a18,18,0,0,0,14.65-28.46ZM91.24,92.78H25.47A10,10,0,0,1,17.33,77L50.21,30.88a10,10,0,0,1,16.28,0L99.38,77A10,10,0,0,1,91.24,92.78Z"/><path class="cls-3" d="M114.83,109.26,66.56,41.61a10.07,10.07,0,0,0-16.41,0L1.88,109.26a10.08,10.08,0,0,0,8.2,15.93h96.55a10.08,10.08,0,0,0,8.2-15.93Zm-8.2,7.93H10.08a2.08,2.08,0,0,1-1.69-3.29L56.66,46.25a2.08,2.08,0,0,1,1.69-.87,2.05,2.05,0,0,1,1.69.87l48.28,67.65A2.08,2.08,0,0,1,106.63,117.19Z"/></g></svg> \ No newline at end of file diff --git a/datahub-web-react/.storybook/storybook-theme.css b/datahub-web-react/.storybook/storybook-theme.css new file mode 100644 index 00000000000000..edf93c57cf2086 --- /dev/null +++ b/datahub-web-react/.storybook/storybook-theme.css @@ -0,0 +1,263 @@ +/* Storybook Theme CSS Overrides */ + +/* Regular */ +@font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 400; + src: url('../src/fonts/Mulish-Regular.ttf') format('truetype'); +} + +/* Medium */ +@font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 500; + src: url('../src/fonts/Mulish-Medium.ttf') format('truetype'); +} + +/* SemiBold */ +@font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 600; + src: url('../src/fonts/Mulish-SemiBold.ttf') format('truetype'); +} + +/* Bold */ +@font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 700; + src: url('../src/fonts/Mulish-Bold.ttf') format('truetype'); +} + +body { + font-family: 'Mulish', sans-serif !important; +} + +::-webkit-scrollbar { + height: 8px; + width: 8px; +} + +*::-webkit-scrollbar-track { + background: rgba(193, 196, 208, 0); + border-radius: 10px; +} + +*::-webkit-scrollbar-thumb { + background: rgba(193, 196, 208, 0); + border-radius: 10px; + transition: 0.3s; +} + +*:hover::-webkit-scrollbar-track { + background: rgba(193, 196, 208, 0.3); +} + +*:hover::-webkit-scrollbar-thumb { + background: rgba(193, 196, 208, 0.8); +} + +.sbdocs-wrapper { + max-width: 95% !important; +} + +.sidebar-header img { + max-height: 25px !important; +} + +.sb-bar { + box-shadow: none !important; + border-bottom: 1px solid hsla(203, 50%, 30%, 0.15) !important; +} + +.sbdocs-preview, +.docblock-argstable-body, +.docblock-source { + box-shadow: none !important; + filter: none !important; +} + +.docblock-source { + max-width: 100% !important; + overflow: auto !important; + margin: 1rem 0 !important; +} + +.sidebar-item, +.sidebar-item[data-selected="true"] { + height: 32px !important; + display: flex !important; + align-items: center !important; + padding-right: 0 !important; + padding: 6px 12px !important; + font-size: 15px !important; + margin-bottom: 4px !important; + color: #000 !important; +} + +.sidebar-item:hover { + background-color: #eff8fc !important; +} + +.sidebar-item>a { + align-items: center !important; + gap: 8px !important; + padding: 0 !important; +} + +.sidebar-item[data-nodetype="group"] { + margin-top: 8px !important; +} + +.sidebar-item[data-nodetype="component"] { + padding-left: 8px !important; +} + +[data-nodetype="root"]>[data-action="collapse-root"]>div:first-child, +[data-nodetype="component"] div { + display: none !important; +} + +[data-nodetype="document"][data-parent-id], +[data-nodetype="story"][data-parent-id] { + padding: 0 !important; + margin-left: 16px !important; + height: 18px !important; + min-height: auto !important; + font-weight: 400 !important; +} + +[data-nodetype="document"][data-parent-id] svg, +[data-nodetype="story"][data-parent-id] svg { + display: none !important; +} + +[data-nodetype="document"][data-parent-id]::before, +[data-nodetype="story"][data-parent-id]::before { + content: '→' !important; +} + +[data-nodetype="document"][data-parent-id]:hover, +[data-nodetype="story"][data-parent-id]:hover, +[data-nodetype="document"][data-parent-id][data-selected="true"]:hover, +[data-nodetype="story"][data-parent-id][data-selected="true"]:hover { + background-color: #fff !important; + color: #4da1bf !important; +} + +[data-nodetype="document"][data-parent-id][data-selected="true"], +[data-nodetype="story"][data-parent-id][data-selected="true"] { + background-color: #fff !important; + height: 18px !important; + min-height: auto !important; + font-weight: 400 !important; +} + +.sbdocs-content div[id*=--sandbox]~div[id*=--sandbox]~div[id*=--sandbox], +li:has(a[href="#sandbox"]) { + display: none !important; +} + +[data-nodetype="document"]:not([data-parent-id]) { + padding-left: 0 !important; +} + +[data-nodetype="document"]:not([data-parent-id]) svg { + display: none !important; +} + +[data-nodetype="document"]:not([data-parent-id])>a { + font-size: 18px !important; + font-weight: 300 !important; +} + +[data-nodetype="component"][aria-expanded="true"], +[data-nodetype="document"][data-selected="true"] { + color: #000 !important; + background-color: transparent !important; + font-weight: 700 !important; +} + +[data-nodetype="root"][data-selected="true"] { + background-color: transparent !important; +} + +[data-nodetype="document"][data-selected="true"], +[data-nodetype="document"][data-parent-id][data-selected="true"] { + color: #4da1bf !important; +} + +.sidebar-subheading { + font-size: 12px !important; + font-weight: 600 !important; + letter-spacing: 1px !important; + color: #a9adbd !important; +} + +.sbdocs-wrapper { + padding: 2rem !important; +} + +table, +tr, +tbody>tr>* { + border-color: hsla(203, 50%, 30%, 0.15) !important; + background-color: transparent; +} + +:where(table:not(.sb-anchor, .sb-unstyled, .sb-unstyled table)) tr:nth-of-type(2n) { + background-color: transparent !important; +} + +tr { + border-top: 0 !important; +} + +th { + border: 0 !important; +} + +h2#stories { + display: none; +} + +.tabbutton { + border-bottom: none !important +} + +.tabbutton.tabbutton-active { + color: rgb(120, 201, 230) !important; +} + +.toc-wrapper { + margin-top: -2.5rem !important; + font-family: 'Mulish', sans-serif !important; +} + +/* Custom Doc Styles */ + +.custom-docs { + position: relative; +} + +.acrylBg { + position: fixed; + bottom: 0; + left: -20px; + background-repeat: repeat; + z-index: 0; +} + +.acrylBg img { + filter: invert(8); +} + +.custom-docs p, +.docsDescription p, +.custom-docs li { + font-size: 16px; + line-height: 1.75; +} \ No newline at end of file diff --git a/datahub-web-react/.storybook/storybook-theme.js b/datahub-web-react/.storybook/storybook-theme.js new file mode 100644 index 00000000000000..462bf2f03da944 --- /dev/null +++ b/datahub-web-react/.storybook/storybook-theme.js @@ -0,0 +1,47 @@ +import { create } from '@storybook/theming'; +import brandImage from './storybook-logo.svg'; + +import theme, { typography } from '../src/alchemy-components/theme'; + +export default create({ + // config + base: 'light', + brandTitle: 'Acryl Design System', + brandUrl: '/?path=/docs/', + brandImage: brandImage, + brandTarget: '_self', + + // styles + fontBase: typography.fontFamily, + fontCode: 'monospace', + + colorPrimary: theme.semanticTokens.colors.primary, + colorSecondary: theme.semanticTokens.colors.secondary, + + // UI + appBg: theme.semanticTokens.colors['body-bg'], + appContentBg: theme.semanticTokens.colors['body-bg'], + appPreviewBg: theme.semanticTokens.colors['body-bg'], + appBorderColor: theme.semanticTokens.colors['border-color'], + appBorderRadius: 4, + + // Text colors + textColor: theme.semanticTokens.colors['body-text'], + textInverseColor: theme.semanticTokens.colors['inverse-text'], + textMutedColor: theme.semanticTokens.colors['subtle-text'], + + // Toolbar default and active colors + barTextColor: theme.semanticTokens.colors['body-text'], + barSelectedColor: theme.semanticTokens.colors['subtle-bg'], + barHoverColor: theme.semanticTokens.colors['subtle-bg'], + barBg: theme.semanticTokens.colors['body-bg'], + + // Form colors + inputBg: theme.semanticTokens.colors['body-bg'], + inputBorder: theme.semanticTokens.colors['border-color'], + inputTextColor: theme.semanticTokens.colors['body-text'], + inputBorderRadius: 4, + + // Grid + gridCellSize: 6, +}); \ No newline at end of file diff --git a/datahub-web-react/.storybook/styledComponents.ts b/datahub-web-react/.storybook/styledComponents.ts new file mode 100644 index 00000000000000..5951c810d89985 --- /dev/null +++ b/datahub-web-react/.storybook/styledComponents.ts @@ -0,0 +1,36 @@ +import { createGlobalStyle } from 'styled-components'; + +import '../src/fonts/Mulish-Regular.ttf'; +import '../src/fonts/Mulish-Medium.ttf'; +import '../src/fonts/Mulish-SemiBold.ttf'; +import '../src/fonts/Mulish-Bold.ttf'; + +export const GlobalStyle = createGlobalStyle` + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 400; + src: url('../src/fonts/Mulish-Regular.ttf) format('truetype'); + } + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 500; + src: url('../src/fonts/Mulish-Medium.ttf) format('truetype'); + } + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 600; + src: url('../src/fonts/Mulish-SemiBold.ttf) format('truetype'); + } + @font-face { + font-family: 'Mulish'; + font-style: normal; + font-weight: 700; + src: url('../src/fonts/Mulish-Bold.ttf) format('truetype'); + } + body { + font-family: 'Mulish', sans-serif; + } +`; \ No newline at end of file diff --git a/datahub-web-react/.storybook/webpack.config.js b/datahub-web-react/.storybook/webpack.config.js new file mode 100644 index 00000000000000..22e4ec1de63050 --- /dev/null +++ b/datahub-web-react/.storybook/webpack.config.js @@ -0,0 +1,13 @@ +const path = require('path'); + +module.exports = { + module: { + loaders: [ + { + test: /\.(png|woff|woff2|eot|ttf|svg)$/, + loaders: ['file-loader'], + include: path.resolve(__dirname, '../'), + }, + ], + }, +}; \ No newline at end of file diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index dcaef6004d7022..31c10804482f0c 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -9,8 +9,12 @@ "@ant-design/colors": "^5.0.0", "@ant-design/icons": "^4.3.0", "@apollo/client": "^3.3.19", + "@fontsource/mulish": "^5.0.16", + "@geometricpanda/storybook-addon-badges": "^2.0.2", "@graphql-codegen/fragment-matcher": "^5.0.0", "@monaco-editor/react": "^4.3.1", + "@mui/icons-material": "^5.15.21", + "@mui/material": "^5.15.21", "@react-hook/window-size": "^3.0.7", "@react-spring/web": "^9.7.3", "@remirror/pm": "^2.0.3", @@ -30,6 +34,7 @@ "@uiw/react-md-editor": "^3.3.4", "@visx/axis": "^3.1.0", "@visx/curve": "^3.0.0", + "@visx/gradient": "^3.3.0", "@visx/group": "^3.0.0", "@visx/hierarchy": "^3.0.0", "@visx/legend": "^3.2.0", @@ -93,7 +98,9 @@ "format-check": "prettier --check src", "format": "prettier --write src", "type-check": "tsc --noEmit", - "type-watch": "tsc -w --noEmit" + "type-watch": "tsc -w --noEmit", + "storybook": "storybook dev -p 6006", + "build-storybook": "storybook build" }, "browserslist": { "production": [ @@ -112,6 +119,16 @@ "@graphql-codegen/near-operation-file-preset": "^1.17.13", "@graphql-codegen/typescript-operations": "1.17.13", "@graphql-codegen/typescript-react-apollo": "2.2.1", + "@storybook/addon-essentials": "^8.1.11", + "@storybook/addon-interactions": "^8.1.11", + "@storybook/addon-links": "^8.1.11", + "@storybook/addon-onboarding": "^8.1.11", + "@storybook/blocks": "^8.1.11", + "@storybook/builder-vite": "^8.1.11", + "@storybook/manager-api": "^8.1.11", + "@storybook/react-vite": "^8.1.11", + "@storybook/test": "^8.1.11", + "@storybook/theming": "^8.1.11", "@types/graphql": "^14.5.0", "@types/query-string": "^6.3.0", "@types/styled-components": "^5.1.7", @@ -132,6 +149,7 @@ "less": "^4.2.0", "prettier": "^2.8.8", "source-map-explorer": "^2.5.2", + "storybook": "^8.1.11", "vite": "^4.5.5", "vite-plugin-babel-macros": "^1.0.6", "vite-plugin-static-copy": "^0.17.0", diff --git a/datahub-web-react/src/alchemy-components/.docs/Contributing.mdx b/datahub-web-react/src/alchemy-components/.docs/Contributing.mdx new file mode 100644 index 00000000000000..75a31d011903f8 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/Contributing.mdx @@ -0,0 +1,43 @@ +import { Meta } from '@storybook/blocks'; + +<Meta title="Contributing" /> + +<div className="custom-docs"> + ## Contributing + + Building and maintinging a design system is a collaborative effort. We welcome contributions from all team members, regardless of their role or experience level. This document outlines the process for contributing to the Acryl Component Library. + + ### Development + + To run Storybook locally, use the following command: + + ``` + yarn storybook + ``` + + Storybook will start a local development server and open a new browser window with the Storybook interface on port `6006`. When developing new components or updating existing ones, you can use Storybook to preview your changes in real-time. This will ensure that the component looks and behaves as expected before merging your changes. + + ### Crafting New Components + + When creating new components, make sure to follow the established design patterns and coding standards. This will help maintain consistency across all Acryl products and make it easier for other team members to understand and use your components. + + Design new components with <strong>reusability in mind</strong>. Components should be flexible, extensible, and easy to customize. Avoid hardcoding values and use props to pass data and styles to your components. This will make it easier to reuse the component in different contexts and scenarios. + + Our design team works exclusively in Figma, so if questions arise about the design or implementation of a component, please refer to the Figma files for more information. If you have any questions or need clarification, feel free to reach out to the design team for assistance. + + ### Pull Requests + + When submitting a pull request, please follow these guidelines: + + 1. Create a new branch for your changes. + 2. Make sure your code is well-documented and follows the established coding standards. + 3. Write clear and concise commit messages. + 4. Include a detailed description of the changes in your pull request. + + If applicable, include screenshots or GIFs to demonstrate the changes visually. This will help reviewers understand the context of your changes and provide more accurate feedback. If a Figma file exists, include a link to the file in the pull request description. + + ### Review Process + + All pull requests will be reviewed by the UI and design team to ensure that the changes align with the design system guidelines and best practices. The team will provide feedback and suggestions for improvement, and you may be asked to make additional changes before your pull request is merged. + +</div> diff --git a/datahub-web-react/src/alchemy-components/.docs/DesignTokens.mdx b/datahub-web-react/src/alchemy-components/.docs/DesignTokens.mdx new file mode 100644 index 00000000000000..0ebdebbf9db4cb --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/DesignTokens.mdx @@ -0,0 +1,63 @@ +import { Meta, Source } from '@storybook/blocks'; + +import theme from '@components/theme'; + +import { ColorCard, CopyButton } from './mdx-components'; + +<Meta title="Design Tokens" /> + +<div className="custom-docs"> + ## Design Tokens + + To streamline the design process and ensure consistency across all Acryl products, we use a set of design tokens that define the visual properties of our design system. These tokens include colors, typography, spacing, and other visual elements that can be used to create a cohesive user experience. + + ### Colors + + ```tsx + import theme from '@components/theme'; + + // Accessing a color via object path + <div style={{ color: theme.semanticTokens.colors.primary }}>Hello, World!</div> + + // Using CSS variables + <div style={{ color: 'var(--alch-color-primary)' }}>Hello, World!</div> + ``` + + <table style={{ width: '100%' }}> + <thead style={{ textAlign: 'left' }}> + <tr> + <th>Token Value</th> + <th>Selector</th> + <th>CSS Variable <small>(coming soon)</small></th> + </tr> + </thead> + <tbody> + {Object.keys(theme.semanticTokens.colors).map((color) => { + const objectKey = `colors['${color}']`; + const hexValue = theme.semanticTokens.colors[color]; + const cssVar = `--alch-color-${color}`; + + return ( + <tr key={color}> + <td> + <ColorCard color={hexValue} size="sm"> + <span className="colorChip" /> + <div> + <span className="colorValue">{color}</span> + <span className="hex">{hexValue}</span> + </div> + </ColorCard> + </td> + <td> + <span style={{ display: 'flex', alignItems: 'center', fontSize: 'inherit' }}> + {objectKey} <CopyButton text={objectKey} /> + </span> + </td> + <td>{cssVar}</td> + </tr> + ); + })} + </tbody> + </table> + +</div> diff --git a/datahub-web-react/src/alchemy-components/.docs/Icons.mdx b/datahub-web-react/src/alchemy-components/.docs/Icons.mdx new file mode 100644 index 00000000000000..e3f6ab68461196 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/Icons.mdx @@ -0,0 +1,34 @@ +import { Meta, Source } from '@storybook/blocks'; + +import { AVAILABLE_ICONS } from '@components'; +import { IconGalleryWithSearch } from './mdx-components'; + +<Meta title="Icons" /> + +<div className="custom-docs"> + ## Icons + + Under the hood, we're utilizing the Material Design Icon Library. However, we've crafted out own resuable component to make it easier to use these icons in our application. + + <a href="/?path=/docs/media-icon--docs"> + View the component documentation to learn more + </a> + + In addition to using Materials Design Icons, we've also added a few custom icons to the library. You can access them through the same `<Icon />` component and are represented in the list of available options below. + + ```tsx + import { Icon } from '@components'; + + <Icon icon="AccountCircle" /> + ``` + + <br /> + + ### Gallery + + There are {AVAILABLE_ICONS.length} icons available. <br /> + Name values populate the `icon` prop on the `<Icon />` component. + + <IconGalleryWithSearch icons={AVAILABLE_ICONS} /> + +</div> diff --git a/datahub-web-react/src/alchemy-components/.docs/Intro.mdx b/datahub-web-react/src/alchemy-components/.docs/Intro.mdx new file mode 100644 index 00000000000000..f81d08059c7b44 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/Intro.mdx @@ -0,0 +1,14 @@ +import { Meta, Description } from '@storybook/blocks'; +import ReadMe from '../README.mdx'; + +<Meta title="Introduction" /> + +<div className="custom-docs"> + <div className="acrylBg"> + <img src="https://www.acryldata.io/images/logo-pattern.svg" alt="Acryl Logo" /> + </div> + + {/* To simply, we're rendering the root readme here */} + <ReadMe /> + +</div> diff --git a/datahub-web-react/src/alchemy-components/.docs/StyleGuide.mdx b/datahub-web-react/src/alchemy-components/.docs/StyleGuide.mdx new file mode 100644 index 00000000000000..43199cbbca62d1 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/StyleGuide.mdx @@ -0,0 +1,209 @@ +import { Meta, Source } from '@storybook/blocks'; + +import { Heading } from '@components'; +import { colors } from '@components/theme'; + +import { Grid, FlexGrid, ColorCard, CopyButton, Seperator } from './mdx-components'; + +import borderSource from '@components/theme/foundations/borders?raw'; +import colorsSource from '@components/theme/foundations/colors?raw'; +import typographySource from '@components/theme/foundations/typography?raw'; +import radiusSource from '@components/theme/foundations/radius?raw'; +import shadowsSource from '@components/theme/foundations/shadows?raw'; +import sizesSource from '@components/theme/foundations/sizes?raw'; +import spacingSource from '@components/theme/foundations/spacing?raw'; +import transitionSource from '@components/theme/foundations/transition?raw'; +import zIndexSource from '@components/theme/foundations/zIndex?raw'; + +<Meta title="Style Guide" /> + +<div className="custom-docs"> + ## Style Guide + + The purpose of this Style Guide is to establish a unified and cohesive design language that ensures a consistent user experience across all Acryl products. By adhering to these guidelines, we can maintain a high standard of design quality and improve the usability of our applications. + + ### Theme + + You can import the theme object into any component or file in your application and use it to style your components. The theme object is a single source of truth for your application's design system. + + ```tsx + import { typography, colors, spacing } from '@components/theme'; + ``` + + ### Colors + + Colors are managed via the `colors.ts` file in the `theme/foundations` directory. The colors are defined as a nested object with the following structure: + + <Source code={colorsSource} /> + + By default, all `500` values are considered the "default" value of that color range. For example, `gray.500` is the default gray color. The other values are used for shading and highlighting. Color values are defined in hex format and their values range between 25 and 1000. With 25 being the lighest and 1000 being the darkest. + + #### Black & White + <FlexGrid> + <ColorCard color={colors['black']}> + <span className="colorChip" /> + <div> + <span className="colorValue">Black</span> + <span className="hex">{colors['black']}</span> + </div> + </ColorCard> + <ColorCard color={colors['white']}> + <span className="colorChip" /> + <div> + <span className="colorValue">White</span> + <span className="hex">{colors['white']}</span> + </div> + </ColorCard> + </FlexGrid> + + <Seperator /> + + #### Gray + <Grid> + {Object.keys(colors.gray).map((color) => ( + <ColorCard key={color} color={colors['gray'][color]}> + <span className="colorChip" /> + <div> + <span className="colorValue"> + Gray {color} <CopyButton text={`gray.${color}`} /> + </span> + <span className="hex">{colors['gray'][color]}</span> + </div> + </ColorCard> + ))} + </Grid> + + <Seperator /> + + #### Violet (Primary) + <Grid> + {Object.keys(colors.violet).map((color) => ( + <ColorCard key={color} color={colors['violet'][color]}> + <span className="colorChip" /> + <div> + <span className="colorValue"> + Violet {color} <CopyButton text={`violet.${color}`} /> + </span> + <span className="hex">{colors['violet'][color]}</span> + </div> + </ColorCard> + ))} + </Grid> + + <Seperator /> + + #### Blue + <Grid> + {Object.keys(colors.blue).map((color) => ( + <ColorCard key={color} color={colors['blue'][color]}> + <span className="colorChip" /> + <div> + <span className="colorValue"> + Blue {color} <CopyButton text={`blue.${color}`} /> + </span> + <span className="hex">{colors['blue'][color]}</span> + </div> + </ColorCard> + ))} + </Grid> + + <Seperator /> + + #### Green + <Grid> + {Object.keys(colors.green).map((color) => ( + <ColorCard key={color} color={colors['green'][color]}> + <span className="colorChip" /> + <div> + <span className="colorValue"> + Green {color} <CopyButton text={`green.${color}`} /> + </span> + <span className="hex">{colors['green'][color]}</span> + </div> + </ColorCard> + ))} + </Grid> + + <Seperator /> + + #### Yellow + <Grid> + {Object.keys(colors.yellow).map((color) => ( + <ColorCard key={color} color={colors['yellow'][color]}> + <span className="colorChip" /> + <div> + <span className="colorValue"> + Yellow {color} <CopyButton text={`yellow.${color}`} /> + </span> + <span className="hex">{colors['yellow'][color]}</span> + </div> + </ColorCard> + ))} + </Grid> + + <Seperator /> + + #### Red + <Grid> + {Object.keys(colors.red).map((color) => ( + <ColorCard key={color} color={colors['red'][color]}> + <span className="colorChip" /> + <div> + <span className="colorValue"> + Red {color} <CopyButton text={`red.${color}`} /> + </span> + <span className="hex">{colors['red'][color]}</span> + </div> + </ColorCard> + ))} + </Grid> + + ### Typography + + Font styles are managed via the `typography.ts` file in the `theme/foundations` directory. The primary font family in use is `Mulish`. The font styles are defined as a nested object with the following structure: + + <Source code={typographySource} /> + + ### Borders + + A set of border values defined by the border key. + + <Source code={borderSource} /> + + ### Border Radius + + A set smooth corner radius values defined by the radii key. + + <Source code={radiusSource} /> + + ### Shadows + + A set of shadow values defined by the shadows key. + + <Source code={shadowsSource} /> + + ## Sizes + + A set of size values defined by the sizes key. + + <Source code={sizesSource} /> + + ### Spacing + + A set of spacing values defined by the spacing key. + + <Source code={spacingSource} /> + + ### Transitions + + A set of transition values defined by the transition key. + + <Source code={transitionSource} /> + + ### Z-Index + + A set of z-index values defined by the zindex key. + + <Source code={zIndexSource} /> + +</div> diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/CodeBlock.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CodeBlock.tsx new file mode 100644 index 00000000000000..43b9ebfae64149 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CodeBlock.tsx @@ -0,0 +1,24 @@ +import React from 'react'; + +import { Source, DocsContext } from '@storybook/blocks'; + +export const CodeBlock = () => { + const context = React.useContext(DocsContext); + + const { primaryStory } = context as any; + const component = context ? primaryStory.component.__docgenInfo.displayName : ''; + + if (!context || !primaryStory) return null; + + return ( + <div> + <Source + code={` + import { ${component} } from '@components'; + `} + format + dark + /> + </div> + ); +}; diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/CopyButton.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CopyButton.tsx new file mode 100644 index 00000000000000..c81aa6ed442892 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/CopyButton.tsx @@ -0,0 +1,16 @@ +import React from 'react'; + +import { Button, Icon } from '@components'; +import { copyToClipboard } from './utils'; + +interface Props { + text: string; +} + +export const CopyButton = ({ text }: Props) => ( + <div style={{ display: 'inline-block' }}> + <Button variant="text" color="gray" size="sm" onClick={() => copyToClipboard(text)}> + <Icon icon="ContentCopy" size="xs" /> + </Button> + </div> +); diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/GridList.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/GridList.tsx new file mode 100644 index 00000000000000..5cb4bd27e521a4 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/GridList.tsx @@ -0,0 +1,32 @@ +/* + Docs Only Component that helps to display a list of components in a grid layout. +*/ + +import React, { ReactNode } from 'react'; + +const styles = { + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + gap: '8px', +}; + +interface Props { + isVertical?: boolean; + width?: number | string; + children: ReactNode; +} + +export const GridList = ({ isVertical = false, width = '100%', children }: Props) => { + return ( + <div + style={{ + ...styles, + width, + flexDirection: isVertical ? 'column' : 'row', + }} + > + {children} + </div> + ); +}; diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/IconGalleryWithSearch.tsx b/datahub-web-react/src/alchemy-components/.docs/mdx-components/IconGalleryWithSearch.tsx new file mode 100644 index 00000000000000..d8751509bd6a72 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/IconGalleryWithSearch.tsx @@ -0,0 +1,291 @@ +import React, { useState } from 'react'; + +import { Icon, Button, ButtonProps } from '@components'; +import { IconGrid, IconGridItem, IconDisplayBlock } from './components'; + +interface Props { + icons: string[]; +} + +export const IconGalleryWithSearch = ({ icons }: Props) => { + const [iconSet, setIconSet] = useState(icons); + const [search, setSearch] = useState(''); + const [variant, setVariant] = useState('outline'); + + const filteredIcons = iconSet.filter((icon) => icon.toLowerCase().includes(search.toLowerCase())); + + const arrows = [ + 'ArrowBack', + 'ArrowCircleDown', + 'ArrowCircleLeft', + 'ArrowCircleRight', + 'ArrowCircleUp', + 'ArrowDownward', + 'ArrowForward', + 'ArrowOutward', + 'ArrowUpward', + 'CloseFullscreen', + 'Cached', + 'Code', + 'CodeOff', + 'CompareArrows', + 'Compress', + 'ChevronLeft', + 'ChevronRight', + 'DoubleArrow', + 'FastForward', + 'FastRewind', + 'FileDownload', + 'FileUpload', + 'ForkLeft', + 'ForkRight', + 'GetApp', + 'LastPage', + 'Launch', + 'Login', + 'Logout', + 'LowPriority', + 'ManageHistory', + 'Merge', + 'MergeType', + 'MoveUp', + 'MultipleStop', + 'OpenInFull', + 'Outbound', + 'Outbox', + 'Output', + 'PlayArrow', + 'PlayCircle', + 'Publish', + 'ReadMore', + 'ExitToApp', + 'Redo', + 'Refresh', + 'Replay', + 'ReplyAll', + 'Reply', + 'Restore', + 'SaveAlt', + 'Shortcut', + 'SkipNext', + 'SkipPrevious', + 'Start', + 'Straight', + 'SubdirectoryArrowLeft', + 'SubdirectoryArrowRight', + 'SwapHoriz', + 'SwapVert', + 'SwitchLeft', + 'SwitchRight', + 'SyncAlt', + 'SyncDisabled', + 'SyncLock', + 'Sync', + 'Shuffle', + 'SyncProblem', + 'TrendingDown', + 'TrendingFlat', + 'TrendingUp', + 'TurnLeft', + 'TurnRight', + 'TurnSlightLeft', + 'TurnSlightRight', + 'Undo', + 'UnfoldLessDouble', + 'UnfoldLess', + 'UnfoldMoreDouble', + 'UnfoldMore', + 'UpdateDisabled', + 'Update', + 'Upgrade', + 'Upload', + 'ZoomInMap', + 'ZoomOutMap', + ]; + + const dataViz = [ + 'AccountTree', + 'Analytics', + 'ArtTrack', + 'Article', + 'BackupTable', + 'BarChart', + 'BubbleChart', + 'Calculate', + 'Equalizer', + 'List', + 'FormatListBulleted', + 'FormatListNumbered', + 'Grading', + 'InsertChart', + 'Hub', + 'Insights', + 'Lan', + 'Leaderboard', + 'LegendToggle', + 'Map', + 'MultilineChart', + 'Nat', + 'PivotTableChart', + 'Poll', + 'Polyline', + 'QueryStats', + 'Radar', + 'Route', + 'Rule', + 'Schema', + 'Sort', + 'SortByAlpha', + 'ShowChart', + 'Source', + 'SsidChart', + 'StackedBarChart', + 'StackedLineChart', + 'Storage', + 'TableChart', + 'TableRows', + 'TableView', + 'Timeline', + 'ViewAgenda', + 'ViewArray', + 'ViewCarousel', + 'ViewColumn', + 'ViewComfy', + 'ViewCompact', + 'ViewCozy', + 'ViewDay', + 'ViewHeadline', + 'ViewKanban', + 'ViewList', + 'ViewModule', + 'ViewQuilt', + 'ViewSidebar', + 'ViewStream', + 'ViewTimeline', + 'ViewWeek', + 'Visibility', + 'VisibilityOff', + 'Webhook', + 'Window', + ]; + + const social = [ + 'AccountCircle', + 'Badge', + 'Campaign', + 'Celebration', + 'Chat', + 'ChatBubble', + 'CommentBank', + 'Comment', + 'CommentsDisabled', + 'Message', + 'ContactPage', + 'Contacts', + 'GroupAdd', + 'Group', + 'GroupRemove', + 'Groups', + 'Handshake', + 'ManageAccounts', + 'MoodBad', + 'SentimentDissatisfied', + 'SentimentNeutral', + 'SentimentSatisfied', + 'Mood', + 'NoAccounts', + 'People', + 'PersonAddAlt1', + 'PersonOff', + 'Person', + 'PersonRemoveAlt1', + 'PersonSearch', + 'SwitchAccount', + 'StarBorder', + 'StarHalf', + 'Star', + 'ThumbDown', + 'ThumbUp', + 'ThumbsUpDown', + 'Verified', + 'VerifiedUser', + ]; + + const notifs = [ + 'Mail', + 'Drafts', + 'MarkAsUnread', + 'Inbox', + 'Outbox', + 'MoveToInbox', + 'Unsubscribe', + 'Upcoming', + 'NotificationAdd', + 'NotificationImportant', + 'NotificationsActive', + 'NotificationsOff', + 'Notifications', + 'NotificationsPaused', + ]; + + const handleChangeSet = (set) => { + setIconSet(set); + setSearch(''); + }; + + const handleResetSet = () => { + setIconSet(icons); + setSearch(''); + }; + + const smButtonProps: ButtonProps = { + size: 'sm', + color: 'gray', + }; + + return ( + <> + <input + type="search" + value={search} + onChange={(e) => setSearch(e.target.value)} + placeholder="Search for an icon…" + style={{ width: '100%', padding: '0.5rem', marginBottom: '0.5rem' }} + /> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: '8px' }}> + <div style={{ display: 'flex', gap: '8px' }}> + <Button onClick={handleResetSet} {...smButtonProps}> + All + </Button> + <Button onClick={() => handleChangeSet(arrows)} {...smButtonProps}> + Arrows + </Button> + <Button onClick={() => handleChangeSet(dataViz)} {...smButtonProps}> + Data Viz + </Button> + <Button onClick={() => handleChangeSet(social)} {...smButtonProps}> + Social + </Button> + <Button onClick={() => handleChangeSet(notifs)} {...smButtonProps}> + Notifications + </Button> + </div> + <div style={{ display: 'flex', gap: '8px' }}> + <Button onClick={() => setVariant(variant === 'outline' ? 'filled' : 'outline')} {...smButtonProps}> + Variant: {variant === 'filled' ? 'Filled' : 'Outline'} + </Button> + </div> + </div> + <IconGrid> + {filteredIcons.map((icon) => ( + <IconGridItem> + <IconDisplayBlock key={icon} title={icon}> + <Icon icon={icon} variant={variant as any} size="2xl" /> + </IconDisplayBlock> + <span>{icon}</span> + </IconGridItem> + ))} + </IconGrid> + </> + ); +}; diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/components.ts b/datahub-web-react/src/alchemy-components/.docs/mdx-components/components.ts new file mode 100644 index 00000000000000..28d428493b17b2 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/components.ts @@ -0,0 +1,110 @@ +/* + Docs Only Components that helps to display information in info guides. +*/ + +import styled from 'styled-components'; + +import theme from '@components/theme'; + +export const Grid = styled.div` + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 16px; +`; + +export const FlexGrid = styled.div` + display: flex; + gap: 16px; +`; + +export const VerticalFlexGrid = styled.div` + display: flex; + flex-direction: column; + gap: 16px; +`; + +export const Seperator = styled.div` + height: 16px; +`; + +export const ColorCard = styled.div<{ color: string; size?: string }>` + display: flex; + gap: 16px; + align-items: center; + + ${({ size }) => + size === 'sm' && + ` + gap: 8px; + `} + + & span { + display: block; + line-height: 1.3; + } + + & .colorChip { + background: ${({ color }) => color}; + width: 3rem; + height: 3rem; + + ${({ size }) => + size === 'sm' && + ` + width: 2rem; + height: 2rem; + border-radius: 4px; + `} + + border-radius: 8px; + box-shadow: rgba(0, 0, 0, 0.06) 0px 2px 4px 0px inset; + } + + & .colorValue { + display: flex; + align-items: center; + gap: 0; + font-weight: bold; + font-size: 14px; + } + + & .hex { + font-size: 11px; + opacity: 0.5; + text-transform: uppercase; + } +`; + +export const IconGrid = styled.div` + display: grid; + grid-template-columns: repeat(auto-fill, minmax(100px, 1fr)); + gap: 16px; + margin-top: 20px; +`; + +export const IconGridItem = styled.div` + display: flex; + flex-direction: column; + align-items: center; + justify-content: space-between; + + border: 1px solid ${theme.semanticTokens.colors['border-color']}; + border-radius: 8px; + overflow: hidden; + + & span { + width: 100%; + border-top: 1px solid ${theme.semanticTokens.colors['border-color']}; + background-color: ${theme.semanticTokens.colors['subtle-bg']}; + text-align: center; + padding: 4px 8px; + font-size: 10px; + } +`; + +export const IconDisplayBlock = styled.div` + display: flex; + align-items: center; + justify-content: center; + height: 50px; +`; diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/index.ts b/datahub-web-react/src/alchemy-components/.docs/mdx-components/index.ts new file mode 100644 index 00000000000000..d1c1848d1eb378 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/index.ts @@ -0,0 +1,6 @@ +export * from './CodeBlock'; +export * from './CopyButton'; +export * from './GridList'; +export * from './IconGalleryWithSearch'; +export * from './components'; +export * from './utils'; diff --git a/datahub-web-react/src/alchemy-components/.docs/mdx-components/utils.ts b/datahub-web-react/src/alchemy-components/.docs/mdx-components/utils.ts new file mode 100644 index 00000000000000..d4fa47dc9e9674 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/.docs/mdx-components/utils.ts @@ -0,0 +1,15 @@ +/* + Docs related utils +*/ + +/** + * Copies the given text to the clipboard. + * @param {string} text - The text to be copied to the clipboard. + * @returns {Promise<void>} A promise that resolves when the text is copied. + */ +export const copyToClipboard = (text: string) => { + return navigator.clipboard + .writeText(text) + .then(() => console.log(`${text} copied to clipboard`)) + .catch(); +}; diff --git a/datahub-web-react/src/alchemy-components/README.mdx b/datahub-web-react/src/alchemy-components/README.mdx new file mode 100644 index 00000000000000..5373432c0ede03 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/README.mdx @@ -0,0 +1,73 @@ +# Alchemy Component Library + +This is a comprehensive library of accessible and reusable React components that streamlines the development of Acryl's applications and websites. The library offers a diverse range of components that can be easily combined to build complex user interfaces while adhering to accessibility best practices. + +### Component Usage + +It's easy to use the components availble in the library. Simply import the component and use it anywhere you're rendering React components. + +```tsx +import { Button } from '@components'; + +function YourComponent() { + return <Button>Click me!</Button>; +} +``` + +In addition to the components themselves, you can also import their types: + +```tsx +import type { ButtonProps } from '@components'; +``` + +### Theme Usage + +This component library comes with a complete theme utility that pre-defines all of our styling atoms and makes them accessible at `@components/theme`. + +```tsx +import { colors } from '@components/theme'; + +function YourComponent() { + return ( + <div style={{ bgColor: colors.green.400 }}> + This div has a green background! + </div> + ) +} +``` + +You can access the theme types at `@components/theme/types` and the theme config at `@components/theme/config`. + +### Writing Docs + +Our docs are generated using [Storybook](https://storybook.js.org/) and deployed to [Cloudfare](https://www.cloudflare.com/). + +- Storybook config is located at `.storybook` +- Static doc files are located at `alchemy-components/.docs` +- Component stories are located in each component directory: <br/>`alchemy-components/components/Component/Component.stories.tsx` + +Storybook serves as our playground for developing components. You can start it locally: + +```bash +yarn storybook +``` + +This launches the docs app at `localhost:6006` and enables everything you need to quickly develop and document components. + +### Contributing + +Building a component library is a collaboriate effort! We're aiming to provide a first-class experience, so here's a list of the standards we'll be looking for: + +- Consitent prop and variant naming conventions: <br /> + -- `variant` is used to define style types, such as `outline` or `filled`. <br /> + -- `color` is used to define the components color, such as `violet` or `blue`. <br /> + -- `size` is used to define the components size, such as `xs` or `4xl`. <br /> + -- Booleans are prefixed with `is`: `isLoading` or `isDisabled`. +- All style props have a correseponding theme type, ie. `FontSizeOptions`. +- All components have an export of default props. +- Styles are defined using `style objects` instead of `tagged template literals`. +- Stories are organized into the correct directory . + +### FAQs + +- **How are components being styled?** <br />Our components are built using [Styled Components](https://styled-components.com/) that dynamically generate styles based on variant selection. diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.stories.tsx b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.stories.tsx new file mode 100644 index 00000000000000..09d0d37f15421a --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.stories.tsx @@ -0,0 +1,133 @@ +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import { GridList } from '@src/alchemy-components/.docs/mdx-components'; +import { Meta, StoryObj } from '@storybook/react'; +import React from 'react'; +import { Avatar, avatarDefaults } from './Avatar'; + +const IMAGE_URL = + 'https://is1-ssl.mzstatic.com/image/thumb/Purple211/v4/78/cb/e1/78cbe16d-28d9-057e-9f73-524c32eb5fe5/AppIcon-0-0-1x_U007emarketing-0-7-0-85-220.png/512x512bb.jpg'; + +// Auto Docs +const meta = { + title: 'Components / Avatar', + component: Avatar, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'This component allows users to render a user pill with picture and name', + }, + }, + + // Component-level argTypes + argTypes: { + name: { + description: 'Name of the user.', + table: { + defaultValue: { summary: `${avatarDefaults.name}` }, + }, + control: 'text', + }, + imageUrl: { + description: 'URL of the user image.', + control: 'text', + }, + onClick: { + description: 'On click function for the Avatar.', + }, + size: { + description: 'Size of the Avatar.', + table: { + defaultValue: { summary: `${avatarDefaults.size}` }, + }, + control: 'select', + }, + showInPill: { + description: 'Whether Avatar is shown in pill format with name.', + table: { + defaultValue: { summary: `${avatarDefaults.showInPill}` }, + }, + control: 'boolean', + }, + + isOutlined: { + description: 'Whether Avatar is outlined.', + table: { + defaultValue: { summary: `${avatarDefaults.isOutlined}` }, + }, + control: 'boolean', + }, + }, + + // Define defaults + args: { + name: 'John Doe', + size: 'default', + showInPill: false, + isOutlined: false, + }, +} satisfies Meta<typeof Avatar>; + +export default meta; + +// Stories + +type Story = StoryObj<typeof meta>; + +// Basic story is what is displayed 1st in storybook & is used as the code sandbox +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => <Avatar {...props} />, +}; + +export const sizes = () => ( + <GridList> + <Avatar name="John Doe" size="lg" /> + <Avatar name="John Doe" size="md" /> + <Avatar name="John Doe" size="default" /> + <Avatar name="John Doe" size="sm" /> + </GridList> +); + +export const withImage = () => ( + <GridList> + <Avatar name="John Doe" imageUrl={IMAGE_URL} size="lg" /> + <Avatar name="John Doe" imageUrl={IMAGE_URL} size="md" /> + <Avatar name="John Doe" imageUrl={IMAGE_URL} size="default" /> + <Avatar name="John Doe" imageUrl={IMAGE_URL} size="sm" /> + </GridList> +); + +export const pills = () => ( + <GridList isVertical> + <GridList> + <Avatar name="John Doe" size="lg" showInPill /> + <Avatar name="John Doe" size="md" showInPill /> + <Avatar name="John Doe" size="default" showInPill /> + <Avatar name="John Doe" size="sm" showInPill /> + </GridList> + <GridList> + <Avatar name="John Doe" size="lg" imageUrl={IMAGE_URL} showInPill /> + <Avatar name="John Doe" size="md" imageUrl={IMAGE_URL} showInPill /> + <Avatar name="John Doe" size="default" imageUrl={IMAGE_URL} showInPill /> + <Avatar name="John Doe" size="sm" imageUrl={IMAGE_URL} showInPill /> + </GridList> + </GridList> +); + +export const outlined = () => ( + <GridList> + <Avatar name="John Doe" size="lg" imageUrl={IMAGE_URL} isOutlined /> + <Avatar name="John Doe" size="lg" showInPill imageUrl={IMAGE_URL} isOutlined /> + </GridList> +); + +export const withOnClick = () => ( + <GridList> + <Avatar name="John Doe" onClick={() => window.alert('Avatar clicked')} /> + <Avatar name="John Doe" onClick={() => window.alert('Avatar clicked')} showInPill /> + </GridList> +); diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.tsx b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.tsx new file mode 100644 index 00000000000000..9e5ec025e08e3d --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Avatar/Avatar.tsx @@ -0,0 +1,40 @@ +import React, { useState } from 'react'; +import { AvatarImage, AvatarImageWrapper, AvatarText, Container } from './components'; +import { AvatarProps } from './types'; +import getAvatarColor, { getNameInitials } from './utils'; + +export const avatarDefaults: AvatarProps = { + name: 'User name', + size: 'default', + showInPill: false, + isOutlined: false, +}; + +export const Avatar = ({ + name = avatarDefaults.name, + imageUrl, + size = avatarDefaults.size, + onClick, + showInPill = avatarDefaults.showInPill, + isOutlined = avatarDefaults.isOutlined, +}: AvatarProps) => { + const [hasError, setHasError] = useState(false); + + return ( + <Container onClick={onClick} $hasOnClick={!!onClick} $showInPill={showInPill}> + <AvatarImageWrapper + $color={getAvatarColor(name)} + $size={size} + $isOutlined={isOutlined} + $hasImage={!!imageUrl} + > + {!hasError && imageUrl ? ( + <AvatarImage src={imageUrl} onError={() => setHasError(true)} /> + ) : ( + <>{getNameInitials(name)} </> + )} + </AvatarImageWrapper> + {showInPill && <AvatarText $size={size}>{name}</AvatarText>} + </Container> + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/_tests_/getNameInitials.test.ts b/datahub-web-react/src/alchemy-components/components/Avatar/_tests_/getNameInitials.test.ts new file mode 100644 index 00000000000000..54bb258acb0d81 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Avatar/_tests_/getNameInitials.test.ts @@ -0,0 +1,34 @@ +import { getNameInitials } from '../utils'; + +describe('get initials of the name', () => { + it('get initials of name with first name and last name', () => { + expect(getNameInitials('John Doe ')).toEqual('JD'); + }); + it('get initials of name with first name and last name in lower case', () => { + expect(getNameInitials('john doe')).toEqual('JD'); + }); + it('get initials of name with only first name', () => { + expect(getNameInitials('Robert')).toEqual('RO'); + }); + it('get initials of name with only first name in lower case', () => { + expect(getNameInitials('robert')).toEqual('RO'); + }); + it('get initials of name with three names', () => { + expect(getNameInitials('James Edward Brown')).toEqual('JB'); + }); + it('get initials of name with four names', () => { + expect(getNameInitials('Michael James Alexander Scott')).toEqual('MS'); + }); + it('get initials of name with a hyphen', () => { + expect(getNameInitials('Mary-Jane Watson')).toEqual('MW'); + }); + it('get initials of name with an apostrophe', () => { + expect(getNameInitials("O'Connor")).toEqual('OC'); + }); + it('get initials of name with a single letter', () => { + expect(getNameInitials('J')).toEqual('J'); + }); + it('get initials of name with an empty string', () => { + expect(getNameInitials('')).toEqual(''); + }); +}); diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/components.ts b/datahub-web-react/src/alchemy-components/components/Avatar/components.ts new file mode 100644 index 00000000000000..bcd23a8ab086c9 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Avatar/components.ts @@ -0,0 +1,51 @@ +import { colors } from '@src/alchemy-components/theme'; +import { AvatarSizeOptions } from '@src/alchemy-components/theme/config'; +import styled from 'styled-components'; +import { getAvatarColorStyles, getAvatarNameSizes, getAvatarSizes } from './utils'; + +export const Container = styled.div<{ $hasOnClick: boolean; $showInPill?: boolean }>` + display: inline-flex; + align-items: center; + gap: 4px; + border-radius: 20px; + border: ${(props) => props.$showInPill && `1px solid ${colors.gray[100]}`}; + padding: ${(props) => props.$showInPill && '3px 6px 3px 4px'}; + + ${(props) => + props.$hasOnClick && + ` + :hover { + cursor: pointer; + } + `} +`; + +export const AvatarImageWrapper = styled.div<{ + $color: string; + $size?: AvatarSizeOptions; + $isOutlined?: boolean; + $hasImage?: boolean; +}>` + ${(props) => getAvatarSizes(props.$size)} + + border-radius: 50%; + color: ${(props) => props.$color}; + border: ${(props) => props.$isOutlined && `1px solid ${colors.gray[1800]}`}; + display: flex; + align-items: center; + justify-content: center; + ${(props) => !props.$hasImage && getAvatarColorStyles(props.$color)} +`; + +export const AvatarImage = styled.img` + width: 100%; + height: 100%; + object-fit: cover; + border-radius: 50%; +`; + +export const AvatarText = styled.span<{ $size?: AvatarSizeOptions }>` + color: ${colors.gray[1700]}; + font-weight: 600; + font-size: ${(props) => getAvatarNameSizes(props.$size)}; +`; diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/index.ts b/datahub-web-react/src/alchemy-components/components/Avatar/index.ts new file mode 100644 index 00000000000000..d3fb6dfa7c09e1 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Avatar/index.ts @@ -0,0 +1 @@ +export { Avatar } from './Avatar'; diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/types.ts b/datahub-web-react/src/alchemy-components/components/Avatar/types.ts new file mode 100644 index 00000000000000..98c554b620dcbd --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Avatar/types.ts @@ -0,0 +1,10 @@ +import { AvatarSizeOptions } from '@src/alchemy-components/theme/config'; + +export interface AvatarProps { + name: string; + imageUrl?: string; + onClick?: () => void; + size?: AvatarSizeOptions; + showInPill?: boolean; + isOutlined?: boolean; +} diff --git a/datahub-web-react/src/alchemy-components/components/Avatar/utils.ts b/datahub-web-react/src/alchemy-components/components/Avatar/utils.ts new file mode 100644 index 00000000000000..46b2ee25488b89 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Avatar/utils.ts @@ -0,0 +1,64 @@ +import { colors } from '@src/alchemy-components/theme'; + +export const getNameInitials = (userName: string) => { + if (!userName) return ''; + const names = userName.trim().split(/[\s']+/); // Split by spaces or apostrophes + if (names.length === 1) { + const firstName = names[0]; + return firstName.length > 1 ? firstName[0]?.toUpperCase() + firstName[1]?.toUpperCase() : firstName[0]; + } + return names[0][0]?.toUpperCase() + names[names.length - 1][0]?.toUpperCase() || ''; +}; + +export function hashString(str: string) { + let hash = 0; + if (str.length === 0) { + return hash; + } + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + // eslint-disable-next-line + hash = (hash << 5) - hash + char; + // eslint-disable-next-line + hash = hash & hash; // Convert to 32bit integer + } + return Math.abs(hash); +} + +const colorMap = { + [colors.violet[500]]: { backgroundColor: colors.gray[1000], border: `1px solid ${colors.violet[1000]}` }, + [colors.blue[1000]]: { backgroundColor: colors.gray[1100], border: `1px solid ${colors.blue[200]}` }, + [colors.gray[600]]: { backgroundColor: colors.gray[1500], border: `1px solid ${colors.gray[100]}` }, +}; + +const avatarColors = Object.keys(colorMap); + +export const getAvatarColorStyles = (color) => { + return { + ...colorMap[color], + }; +}; + +export default function getAvatarColor(name: string) { + return avatarColors[hashString(name) % avatarColors.length]; +} + +export const getAvatarSizes = (size) => { + const sizeMap = { + sm: { width: '18px', height: '18px', fontSize: '8px' }, + md: { width: '24px', height: '24px', fontSize: '12px' }, + lg: { width: '28px', height: '28px', fontSize: '14px' }, + default: { width: '20px', height: '20px', fontSize: '10px' }, + }; + + return { + ...sizeMap[size], + }; +}; + +export const getAvatarNameSizes = (size) => { + if (size === 'lg') return '16px'; + if (size === 'sm') return '10px'; + if (size === 'md') return '14px'; + return '12px'; +}; diff --git a/datahub-web-react/src/alchemy-components/components/Badge/Badge.stories.tsx b/datahub-web-react/src/alchemy-components/components/Badge/Badge.stories.tsx new file mode 100644 index 00000000000000..88d499226feafd --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Badge/Badge.stories.tsx @@ -0,0 +1,102 @@ +import React from 'react'; + +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import type { Meta, StoryObj } from '@storybook/react'; + +import { GridList } from '@components/.docs/mdx-components'; +import { Badge, badgeDefault } from './Badge'; +import pillMeta from '../Pills/Pill.stories'; +import { omitKeys } from './utils'; + +const pillMetaArgTypes = omitKeys(pillMeta.argTypes, ['label']); +const pillMetaArgs = omitKeys(pillMeta.args, ['label']); + +const meta = { + title: 'Components / Badge', + component: Badge, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.EXPERIMENTAL], + docs: { + subtitle: 'A component that is used to get badge', + }, + }, + + // Component-level argTypes + argTypes: { + count: { + description: 'Count to show.', + table: { + defaultValue: { summary: `${badgeDefault.count}` }, + }, + control: { + type: 'number', + }, + }, + overflowCount: { + description: 'Max count to show.', + table: { + defaultValue: { summary: `${badgeDefault.overflowCount}` }, + }, + control: { + type: 'number', + }, + }, + showZero: { + description: 'Whether to show badge when `count` is zero.', + table: { + defaultValue: { summary: `${badgeDefault.showZero}` }, + }, + control: { + type: 'boolean', + }, + }, + ...pillMetaArgTypes, + }, + + // Define defaults + args: { + count: 100, + overflowCount: badgeDefault.overflowCount, + showZero: badgeDefault.showZero, + ...pillMetaArgs, + }, +} satisfies Meta<typeof Badge>; + +export default meta; + +type Story = StoryObj<typeof meta>; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => <Badge {...props} />, +}; + +export const sizes = () => ( + <GridList> + <Badge count={100} /> + <Badge count={100} size="sm" /> + <Badge count={100} size="lg" /> + </GridList> +); + +export const colors = () => ( + <GridList> + <Badge count={100} /> + <Badge count={100} colorScheme="violet" /> + <Badge count={100} colorScheme="green" /> + <Badge count={100} colorScheme="red" /> + <Badge count={100} colorScheme="blue" /> + <Badge count={100} colorScheme="gray" /> + </GridList> +); + +export const withIcon = () => ( + <GridList> + <Badge count={100} leftIcon="AutoMode" /> + <Badge count={100} rightIcon="Close" /> + <Badge count={100} leftIcon="AutoMode" rightIcon="Close" /> + </GridList> +); diff --git a/datahub-web-react/src/alchemy-components/components/Badge/Badge.tsx b/datahub-web-react/src/alchemy-components/components/Badge/Badge.tsx new file mode 100644 index 00000000000000..1c934ef120eee8 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Badge/Badge.tsx @@ -0,0 +1,29 @@ +import { Pill } from '@components'; +import React, { useMemo } from 'react'; + +import { BadgeProps } from './types'; +import { formatBadgeValue } from './utils'; +import { BadgeContainer } from './components'; + +export const badgeDefault: BadgeProps = { + count: 0, + overflowCount: 99, + showZero: false, +}; + +export function Badge({ + count = badgeDefault.count, + overflowCount = badgeDefault.overflowCount, + showZero = badgeDefault.showZero, + ...props +}: BadgeProps) { + const label = useMemo(() => formatBadgeValue(count, overflowCount), [count, overflowCount]); + + if (!showZero && count === 0) return null; + + return ( + <BadgeContainer title={`${count}`}> + <Pill label={label} {...props} /> + </BadgeContainer> + ); +} diff --git a/datahub-web-react/src/alchemy-components/components/Badge/components.ts b/datahub-web-react/src/alchemy-components/components/Badge/components.ts new file mode 100644 index 00000000000000..a7791cd4f5ff88 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Badge/components.ts @@ -0,0 +1,6 @@ +import styled from 'styled-components'; + +export const BadgeContainer = styled.div({ + // Base root styles + display: 'inline-flex', +}); diff --git a/datahub-web-react/src/alchemy-components/components/Badge/index.ts b/datahub-web-react/src/alchemy-components/components/Badge/index.ts new file mode 100644 index 00000000000000..26a9e305c7ffd5 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Badge/index.ts @@ -0,0 +1 @@ +export { Badge } from './Badge'; diff --git a/datahub-web-react/src/alchemy-components/components/Badge/types.ts b/datahub-web-react/src/alchemy-components/components/Badge/types.ts new file mode 100644 index 00000000000000..21348f2a083419 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Badge/types.ts @@ -0,0 +1,8 @@ +import { HTMLAttributes } from 'react'; +import { PillProps } from '../Pills/types'; + +export interface BadgeProps extends HTMLAttributes<HTMLElement>, Omit<PillProps, 'label'> { + count: number; + overflowCount?: number; + showZero?: boolean; +} diff --git a/datahub-web-react/src/alchemy-components/components/Badge/utils.ts b/datahub-web-react/src/alchemy-components/components/Badge/utils.ts new file mode 100644 index 00000000000000..e59ec2af998e74 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Badge/utils.ts @@ -0,0 +1,15 @@ +export const formatBadgeValue = (value: number, overflowCount?: number): string => { + if (overflowCount === undefined || value < overflowCount) return String(value); + + return `${overflowCount}+`; +}; + +export function omitKeys<T extends object, K extends keyof T>(obj: T, keys: K[]): Omit<T, K> { + const { ...rest } = obj; + + keys.forEach((key) => { + delete rest[key]; + }); + + return rest; +} diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.stories.tsx b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.stories.tsx new file mode 100644 index 00000000000000..1258ff398c0a7e --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.stories.tsx @@ -0,0 +1,90 @@ +import React from 'react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import type { Meta, StoryObj } from '@storybook/react'; +import { BarChart } from './BarChart'; +import { getMockedProps } from './utils'; + +const meta = { + title: 'Charts / BarChart', + component: BarChart, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.EXPERIMENTAL], + docs: { + subtitle: 'A component that is used to show BarChart', + }, + }, + + // Component-level argTypes + argTypes: { + data: { + description: 'Array of datum to show', + }, + xAccessor: { + description: 'A function to convert datum to value of X', + }, + yAccessor: { + description: 'A function to convert datum to value of Y', + }, + renderTooltipContent: { + description: 'A function to replace default rendering of toolbar', + }, + margin: { + description: 'Add margins to chart', + }, + leftAxisTickFormat: { + description: 'A function to format labels of left axis', + }, + leftAxisTickLabelProps: { + description: 'Props for label of left axis', + }, + bottomAxisTickFormat: { + description: 'A function to format labels of bottom axis', + }, + bottomAxisTickLabelProps: { + description: 'Props for label of bottom axis', + }, + barColor: { + description: 'Color of bar', + control: { + type: 'color', + }, + }, + barSelectedColor: { + description: 'Color of selected bar', + control: { + type: 'color', + }, + }, + gridColor: { + description: "Color of grid's lines", + control: { + type: 'color', + }, + }, + renderGradients: { + description: 'A function to render different gradients that can be used as colors', + }, + }, + + // Define defaults + args: { + ...getMockedProps(), + renderTooltipContent: (datum) => <>DATUM: {JSON.stringify(datum)}</>, + }, +} satisfies Meta<typeof BarChart>; + +export default meta; + +type Story = StoryObj<typeof meta>; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => ( + <div style={{ width: '900px', height: '350px' }}> + <BarChart {...props} /> + </div> + ), +}; diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.tsx b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.tsx new file mode 100644 index 00000000000000..eb5465a1d1217b --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/BarChart/BarChart.tsx @@ -0,0 +1,152 @@ +import React, { useState } from 'react'; +import { colors } from '@src/alchemy-components/theme'; +import { TickLabelProps } from '@visx/axis'; +import { LinearGradient } from '@visx/gradient'; +import { ParentSize } from '@visx/responsive'; +import { Axis, AxisScale, BarSeries, Grid, Tooltip, XYChart } from '@visx/xychart'; +import dayjs from 'dayjs'; +import { Popover } from '../Popover'; +import { ChartWrapper, StyledBarSeries } from './components'; +import { BarChartProps } from './types'; +import { abbreviateNumber } from '../dataviz/utils'; + +const commonTickLabelProps: TickLabelProps<any> = { + fontSize: 10, + fontFamily: 'Mulish', + fill: colors.gray[1700], +}; + +export const barChartDefault: BarChartProps<any> = { + data: [], + xAccessor: (datum) => datum?.x, + yAccessor: (datum) => datum?.y, + leftAxisTickFormat: abbreviateNumber, + leftAxisTickLabelProps: { + ...commonTickLabelProps, + textAnchor: 'end', + }, + bottomAxisTickFormat: (value) => dayjs(value).format('DD MMM'), + bottomAxisTickLabelProps: { + ...commonTickLabelProps, + textAnchor: 'middle', + verticalAnchor: 'start', + width: 20, + }, + barColor: 'url(#bar-gradient)', + barSelectedColor: colors.violet[500], + gridColor: '#e0e0e0', + renderGradients: () => <LinearGradient id="bar-gradient" from={colors.violet[500]} to="#917FFF" toOpacity={0.6} />, +}; + +export function BarChart<DatumType extends object = any>({ + data, + xAccessor = barChartDefault.xAccessor, + yAccessor = barChartDefault.yAccessor, + renderTooltipContent, + margin, + leftAxisTickFormat = barChartDefault.leftAxisTickFormat, + leftAxisTickLabelProps = barChartDefault.leftAxisTickLabelProps, + bottomAxisTickFormat = barChartDefault.bottomAxisTickFormat, + bottomAxisTickLabelProps = barChartDefault.bottomAxisTickLabelProps, + barColor = barChartDefault.barColor, + barSelectedColor = barChartDefault.barSelectedColor, + gridColor = barChartDefault.gridColor, + renderGradients = barChartDefault.renderGradients, +}: BarChartProps<DatumType>) { + const [hasSelectedBar, setHasSelectedBar] = useState<boolean>(false); + + // FYI: additional margins to show left and bottom axises + const internalMargin = { + top: (margin?.top ?? 0) + 30, + right: margin?.right ?? 0, + bottom: (margin?.bottom ?? 0) + 35, + left: (margin?.left ?? 0) + 40, + }; + + const accessors = { xAccessor, yAccessor }; + + return ( + <ChartWrapper> + <ParentSize> + {({ width, height }) => { + return ( + <XYChart + width={width} + height={height} + xScale={{ type: 'band', paddingInner: 0.4, paddingOuter: 0.1 }} + yScale={{ type: 'linear', nice: true, round: true }} + margin={internalMargin} + captureEvents={false} + > + {renderGradients?.()} + + <Axis + orientation="left" + hideAxisLine + hideTicks + tickFormat={leftAxisTickFormat} + tickLabelProps={leftAxisTickLabelProps} + /> + + <Axis + orientation="bottom" + numTicks={data.length} + tickFormat={bottomAxisTickFormat} + tickLabelProps={bottomAxisTickLabelProps} + hideAxisLine + hideTicks + /> + + <line + x1={internalMargin.left} + x2={internalMargin.left} + y1={0} + y2={height - internalMargin.bottom} + stroke={gridColor} + /> + + <Grid rows columns={false} stroke={gridColor} strokeWidth={1} lineStyle={{}} /> + + <StyledBarSeries + as={BarSeries<AxisScale, AxisScale, DatumType>} + $hasSelectedItem={hasSelectedBar} + $color={barColor} + $selectedColor={barSelectedColor} + dataKey="bar-seria-0" + data={data} + radius={4} + radiusTop + onBlur={() => setHasSelectedBar(false)} + onFocus={() => setHasSelectedBar(true)} + // Internally the library doesn't emmit these events if handlers are empty + // They are requred to show/hide/move tooltip + onPointerMove={() => null} + onPointerUp={() => null} + onPointerOut={() => null} + {...accessors} + /> + + <Tooltip<DatumType> + snapTooltipToDatumX + snapTooltipToDatumY + unstyled + applyPositionStyle + renderTooltip={({ tooltipData }) => { + return ( + tooltipData?.nearestDatum && ( + <Popover + open + placement="topLeft" + content={renderTooltipContent?.(tooltipData.nearestDatum.datum)} + /> + ) + ); + }} + /> + </XYChart> + ); + }} + </ParentSize> + </ChartWrapper> + ); +} diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/components.tsx b/datahub-web-react/src/alchemy-components/components/BarChart/components.tsx new file mode 100644 index 00000000000000..aa8f1320ef21dd --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/BarChart/components.tsx @@ -0,0 +1,34 @@ +import { colors } from '@src/alchemy-components/theme'; +import { BarSeries } from '@visx/xychart'; +import styled from 'styled-components'; + +export const ChartWrapper = styled.div` + width: 100%; + height: 100%; + position: relative; +`; + +export const StyledBarSeries = styled(BarSeries)<{ + $hasSelectedItem?: boolean; + $color?: string; + $selectedColor?: string; +}>` + & { + cursor: pointer; + + fill: ${(props) => (props.$hasSelectedItem ? props.$selectedColor : props.$color) || colors.violet[500]}; + ${(props) => props.$hasSelectedItem && 'opacity: 0.3;'} + + :hover { + fill: ${(props) => props.$selectedColor || colors.violet[500]}; + filter: drop-shadow(0px -2px 5px rgba(33, 23, 95, 0.3)); + opacity: 1; + } + + :focus { + fill: ${(props) => props.$selectedColor || colors.violet[500]}; + outline: none; + opacity: 1; + } + } +`; diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/index.ts b/datahub-web-react/src/alchemy-components/components/BarChart/index.ts new file mode 100644 index 00000000000000..fdfc3f3ab44a89 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/BarChart/index.ts @@ -0,0 +1 @@ +export { BarChart } from './BarChart'; diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/types.ts b/datahub-web-react/src/alchemy-components/components/BarChart/types.ts new file mode 100644 index 00000000000000..5fd7e2e63e2411 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/BarChart/types.ts @@ -0,0 +1,18 @@ +import { TickFormatter, TickLabelProps } from '@visx/axis'; +import { Margin } from '@visx/xychart'; + +export type BarChartProps<DatumType extends object> = { + data: DatumType[]; + xAccessor: (datum: DatumType) => string | number; + yAccessor: (datum: DatumType) => number; + renderTooltipContent?: (datum: DatumType) => React.ReactNode; + margin?: Margin; + leftAxisTickFormat?: TickFormatter<DatumType>; + leftAxisTickLabelProps?: TickLabelProps<DatumType>; + bottomAxisTickFormat?: TickFormatter<DatumType>; + bottomAxisTickLabelProps?: TickLabelProps<DatumType>; + barColor?: string; + barSelectedColor?: string; + gridColor?: string; + renderGradients?: () => React.ReactNode; +}; diff --git a/datahub-web-react/src/alchemy-components/components/BarChart/utils.ts b/datahub-web-react/src/alchemy-components/components/BarChart/utils.ts new file mode 100644 index 00000000000000..0b592da7f59b08 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/BarChart/utils.ts @@ -0,0 +1,26 @@ +import dayjs from 'dayjs'; + +export function generateMockData(length = 30, maxValue = 50_000, minValue = 0) { + return Array(length) + .fill(0) + .map((_, index) => { + const date = dayjs() + .startOf('day') + .add(index - length, 'days') + .toDate(); + const value = Math.max(Math.random() * maxValue, minValue); + + return { + x: date, + y: value, + }; + }); +} + +export function getMockedProps() { + return { + data: generateMockData(), + xAccessor: (datum) => datum.x, + yAccessor: (datum) => Math.max(datum.y, 1000), + }; +} diff --git a/datahub-web-react/src/alchemy-components/components/Button/Button.stories.tsx b/datahub-web-react/src/alchemy-components/components/Button/Button.stories.tsx new file mode 100644 index 00000000000000..e2d7c2852da519 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Button/Button.stories.tsx @@ -0,0 +1,203 @@ +import React from 'react'; + +import type { Meta, StoryObj } from '@storybook/react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; + +import { GridList } from '@components/.docs/mdx-components'; +import { AVAILABLE_ICONS } from '@components'; + +import { Button, buttonDefaults } from '.'; + +// Auto Docs +const meta = { + title: 'Forms / Button', + component: Button, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: + 'Buttons are used to trigger an action or event, such as submitting a form, opening a dialog, canceling an action, or performing a delete operation.', + }, + }, + + // Component-level argTypes + argTypes: { + children: { + description: 'The content of the Button.', + control: { + type: 'text', + }, + }, + variant: { + description: 'The variant of the Button.', + options: ['filled', 'outline', 'text'], + table: { + defaultValue: { summary: buttonDefaults.variant }, + }, + control: { + type: 'radio', + }, + }, + color: { + description: 'The color of the Button.', + options: ['violet', 'green', 'red', 'blue', 'gray'], + table: { + defaultValue: { summary: buttonDefaults.color }, + }, + control: { + type: 'select', + }, + }, + size: { + description: 'The size of the Button.', + options: ['sm', 'md', 'lg', 'xl'], + table: { + defaultValue: { summary: buttonDefaults.size }, + }, + control: { + type: 'select', + }, + }, + icon: { + description: 'The icon to display in the Button.', + type: 'string', + options: AVAILABLE_ICONS, + table: { + defaultValue: { summary: 'undefined' }, + }, + control: { + type: 'select', + }, + }, + iconPosition: { + description: 'The position of the icon in the Button.', + options: ['left', 'right'], + table: { + defaultValue: { summary: buttonDefaults.iconPosition }, + }, + control: { + type: 'radio', + }, + }, + isCircle: { + description: + 'Whether the Button should be a circle. If this is selected, the Button will ignore children content, so add an Icon to the Button.', + table: { + defaultValue: { summary: buttonDefaults?.isCircle?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isLoading: { + description: 'Whether the Button is in a loading state.', + table: { + defaultValue: { summary: buttonDefaults?.isLoading?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isDisabled: { + description: 'Whether the Button is disabled.', + table: { + defaultValue: { summary: buttonDefaults?.isDisabled?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isActive: { + description: 'Whether the Button is active.', + table: { + defaultValue: { summary: buttonDefaults?.isActive?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + onClick: { + description: 'Function to call when the button is clicked', + table: { + defaultValue: { summary: 'undefined' }, + }, + action: 'clicked', + }, + }, + + // Define defaults + args: { + children: 'Button Content', + variant: buttonDefaults.variant, + color: buttonDefaults.color, + size: buttonDefaults.size, + icon: undefined, + iconPosition: buttonDefaults.iconPosition, + isCircle: buttonDefaults.isCircle, + isLoading: buttonDefaults.isLoading, + isDisabled: buttonDefaults.isDisabled, + isActive: buttonDefaults.isActive, + onClick: () => console.log('Button clicked'), + }, +} satisfies Meta<typeof Button>; + +export default meta; + +// Stories + +type Story = StoryObj<typeof meta>; + +// Basic story is what is displayed 1st in storybook & is used as the code sandbox +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => <Button {...props}>Button</Button>, +}; + +export const states = () => ( + <GridList> + <Button>Default</Button> + <Button isLoading>Loading State</Button> + <Button isActive>Active/Focus State</Button> + <Button isDisabled>Disabled State</Button> + </GridList> +); + +export const colors = () => ( + <GridList> + <Button>Violet Button</Button> + <Button color="green">Green Button</Button> + <Button color="red">Red Button</Button> + <Button color="blue">Blue Button</Button> + <Button color="gray">Gray Button</Button> + </GridList> +); + +export const sizes = () => ( + <GridList> + <Button size="sm">Small Button</Button> + <Button size="md">Regular Button</Button> + <Button size="lg">Large Button</Button> + <Button size="xl">XLarge Button</Button> + </GridList> +); + +export const withIcon = () => ( + <GridList> + <Button icon="Add">Icon Left</Button> + <Button icon="Add" iconPosition="right"> + Icon Right + </Button> + </GridList> +); + +export const circleShape = () => ( + <GridList> + <Button icon="Add" size="sm" isCircle /> + <Button icon="Add" isCircle /> + <Button icon="Add" size="lg" isCircle /> + </GridList> +); diff --git a/datahub-web-react/src/alchemy-components/components/Button/Button.tsx b/datahub-web-react/src/alchemy-components/components/Button/Button.tsx new file mode 100644 index 00000000000000..a727b0faf97a99 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Button/Button.tsx @@ -0,0 +1,60 @@ +import React from 'react'; + +import { LoadingOutlined } from '@ant-design/icons'; + +import { Icon } from '@components'; + +import { ButtonBase } from './components'; +import { ButtonProps } from './types'; + +export const buttonDefaults: ButtonProps = { + variant: 'filled', + color: 'violet', + size: 'md', + iconPosition: 'left', + isCircle: false, + isLoading: false, + isDisabled: false, + isActive: false, +}; + +export const Button = ({ + variant = buttonDefaults.variant, + color = buttonDefaults.color, + size = buttonDefaults.size, + icon, // default undefined + iconPosition = buttonDefaults.iconPosition, + isCircle = buttonDefaults.isCircle, + isLoading = buttonDefaults.isLoading, + isDisabled = buttonDefaults.isDisabled, + isActive = buttonDefaults.isActive, + children, + ...props +}: ButtonProps) => { + const sharedProps = { + variant, + color, + size, + isCircle, + isLoading, + isActive, + isDisabled, + disabled: isDisabled, + }; + + if (isLoading) { + return ( + <ButtonBase {...sharedProps} {...props}> + <LoadingOutlined rotate={10} /> {!isCircle && children} + </ButtonBase> + ); + } + + return ( + <ButtonBase {...sharedProps} {...props}> + {icon && iconPosition === 'left' && <Icon icon={icon} size={size} />} + {!isCircle && children} + {icon && iconPosition === 'right' && <Icon icon={icon} size={size} />} + </ButtonBase> + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Button/components.ts b/datahub-web-react/src/alchemy-components/components/Button/components.ts new file mode 100644 index 00000000000000..49fa9a12ede6e2 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Button/components.ts @@ -0,0 +1,27 @@ +import styled from 'styled-components'; + +import { spacing } from '@components/theme'; +import { ButtonProps } from './types'; +import { getButtonStyle } from './utils'; + +export const ButtonBase = styled.button( + // Dynamic styles + (props: ButtonProps) => ({ ...getButtonStyle(props as ButtonProps) }), + { + // Base root styles + display: 'flex', + alignItems: 'center', + gap: spacing.xsm, + cursor: 'pointer', + transition: `all 0.15s ease`, + + // For transitions between focus/active and hover states + outlineColor: 'transparent', + outlineStyle: 'solid', + + // Base Disabled styles + '&:disabled': { + cursor: 'not-allowed', + }, + }, +); diff --git a/datahub-web-react/src/alchemy-components/components/Button/index.ts b/datahub-web-react/src/alchemy-components/components/Button/index.ts new file mode 100644 index 00000000000000..745d8377f9fbb4 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Button/index.ts @@ -0,0 +1,2 @@ +export { Button, buttonDefaults } from './Button'; +export type { ButtonProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Button/types.ts b/datahub-web-react/src/alchemy-components/components/Button/types.ts new file mode 100644 index 00000000000000..f510ff4c6c13c5 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Button/types.ts @@ -0,0 +1,16 @@ +import { ButtonHTMLAttributes } from 'react'; + +import type { IconNames } from '@components'; +import type { SizeOptions, ColorOptions } from '@components/theme/config'; + +export interface ButtonProps extends ButtonHTMLAttributes<HTMLButtonElement> { + variant?: 'filled' | 'outline' | 'text'; + color?: ColorOptions; + size?: SizeOptions; + icon?: IconNames; + iconPosition?: 'left' | 'right'; + isCircle?: boolean; + isLoading?: boolean; + isDisabled?: boolean; + isActive?: boolean; +} diff --git a/datahub-web-react/src/alchemy-components/components/Button/utils.ts b/datahub-web-react/src/alchemy-components/components/Button/utils.ts new file mode 100644 index 00000000000000..c08f4f067304d1 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Button/utils.ts @@ -0,0 +1,238 @@ +/* + * Button Style Utilities + */ + +import { typography, colors, shadows, radius, spacing } from '@components/theme'; +import { getColor, getFontSize } from '@components/theme/utils'; +import { ButtonProps } from './types'; + +// Utility function to get color styles for button - does not generate CSS +const getButtonColorStyles = (variant, color) => { + const color500 = getColor(color, 500); // value of 500 shade + const isViolet = color === 'violet'; + + const base = { + // Backgrounds + bgColor: color500, + hoverBgColor: getColor(color, 600), + activeBgColor: getColor(color, 700), + disabledBgColor: getColor('gray', 100), + + // Borders + borderColor: color500, + activeBorderColor: getColor(color, 300), + disabledBorderColor: getColor('gray', 200), + + // Text + textColor: colors.white, + disabledTextColor: getColor('gray', 300), + }; + + // Specific color override for white + if (color === 'white') { + base.textColor = colors.black; + base.disabledTextColor = getColor('gray', 500); + } + + // Specific color override for gray + if (color === 'gray') { + base.textColor = getColor('gray', 500); + base.bgColor = getColor('gray', 100); + base.borderColor = getColor('gray', 100); + + base.hoverBgColor = getColor('gray', 100); + base.activeBgColor = getColor('gray', 200); + } + + // Override styles for outline variant + if (variant === 'outline') { + return { + ...base, + bgColor: colors.transparent, + borderColor: color500, + textColor: color500, + + hoverBgColor: isViolet ? getColor(color, 100) : getColor(color, 100), + activeBgColor: isViolet ? getColor(color, 100) : getColor(color, 200), + + disabledBgColor: 'transparent', + }; + } + + // Override styles for text variant + if (variant === 'text') { + return { + ...base, + textColor: color500, + + bgColor: colors.transparent, + borderColor: colors.transparent, + hoverBgColor: colors.transparent, + activeBgColor: colors.transparent, + disabledBgColor: colors.transparent, + disabledBorderColor: colors.transparent, + }; + } + + // Filled variable is the base style + return base; +}; + +// Generate color styles for button +const getButtonVariantStyles = (variant, color) => { + const variantStyles = { + filled: { + backgroundColor: color.bgColor, + border: `1px solid ${color.borderColor}`, + color: color.textColor, + '&:hover': { + backgroundColor: color.hoverBgColor, + border: `1px solid ${color.hoverBgColor}`, + boxShadow: shadows.sm, + }, + '&:disabled': { + backgroundColor: color.disabledBgColor, + border: `1px solid ${color.disabledBorderColor}`, + color: color.disabledTextColor, + boxShadow: shadows.xs, + }, + }, + outline: { + backgroundColor: 'transparent', + border: `1px solid ${color.borderColor}`, + color: color.textColor, + '&:hover': { + backgroundColor: color.hoverBgColor, + boxShadow: 'none', + }, + '&:disabled': { + backgroundColor: color.disabledBgColor, + border: `1px solid ${color.disabledBorderColor}`, + color: color.disabledTextColor, + boxShadow: shadows.xs, + }, + }, + text: { + backgroundColor: 'transparent', + border: 'none', + color: color.textColor, + '&:hover': { + backgroundColor: color.hoverBgColor, + }, + '&:disabled': { + backgroundColor: color.disabledBgColor, + color: color.disabledTextColor, + }, + }, + }; + + return variantStyles[variant]; +}; + +// Generate font styles for button +const getButtonFontStyles = (size) => { + const baseFontStyles = { + fontFamily: typography.fonts.body, + fontWeight: typography.fontWeights.normal, + lineHeight: typography.lineHeights.none, + }; + + const sizeStyles = { + sm: { + ...baseFontStyles, + fontSize: getFontSize(size), // 12px + }, + md: { + ...baseFontStyles, + fontSize: getFontSize(size), // 14px + }, + lg: { + ...baseFontStyles, + fontSize: getFontSize(size), // 16px + }, + xl: { + ...baseFontStyles, + fontSize: getFontSize(size), // 18px + }, + }; + + return sizeStyles[size]; +}; + +// Generate radii styles for button +const getButtonRadiiStyles = (isCircle) => { + if (isCircle) return { borderRadius: radius.full }; + return { borderRadius: radius.sm }; // radius is the same for all button sizes +}; + +// Generate padding styles for button +const getButtonPadding = (size, isCircle) => { + if (isCircle) return { padding: spacing.xsm }; + + const paddingStyles = { + sm: { + padding: '8px 12px', + }, + md: { + padding: '10px 12px', + }, + lg: { + padding: '10px 16px', + }, + xl: { + padding: '12px 20px', + }, + }; + + return paddingStyles[size]; +}; + +// Generate active styles for button +const getButtonActiveStyles = (styleColors) => ({ + borderColor: 'transparent', + backgroundColor: styleColors.activeBgColor, + // TODO: Figure out how to make the #fff interior border transparent + boxShadow: `0 0 0 2px #fff, 0 0 0 4px ${styleColors.activeBgColor}`, +}); + +// Generate loading styles for button +const getButtonLoadingStyles = () => ({ + pointerEvents: 'none', + opacity: 0.75, +}); + +/* + * Main function to generate styles for button + */ +export const getButtonStyle = (props: ButtonProps) => { + const { variant, color, size, isCircle, isActive, isLoading } = props; + + // Get map of colors + const colorStyles = getButtonColorStyles(variant, color) || ({} as any); + + // Define styles for button + const variantStyles = getButtonVariantStyles(variant, colorStyles); + const fontStyles = getButtonFontStyles(size); + const radiiStyles = getButtonRadiiStyles(isCircle); + const paddingStyles = getButtonPadding(size, isCircle); + + // Base of all generated styles + let styles = { + ...variantStyles, + ...fontStyles, + ...radiiStyles, + ...paddingStyles, + }; + + // Focus & Active styles are the same, but active styles are applied conditionally & override prevs styles + const activeStyles = { ...getButtonActiveStyles(colorStyles) }; + styles['&:focus'] = activeStyles; + styles['&:active'] = activeStyles; + if (isActive) styles = { ...styles, ...activeStyles }; + + // Loading styles + if (isLoading) styles = { ...styles, ...getButtonLoadingStyles() }; + + // Return generated styles + return styles; +}; diff --git a/datahub-web-react/src/alchemy-components/components/Card/Card.stories.tsx b/datahub-web-react/src/alchemy-components/components/Card/Card.stories.tsx new file mode 100644 index 00000000000000..336831fd15cfab --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Card/Card.stories.tsx @@ -0,0 +1,141 @@ +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import { GridList } from '@src/alchemy-components/.docs/mdx-components'; +import { colors } from '@src/alchemy-components/theme'; +import type { Meta, StoryObj } from '@storybook/react'; +import React from 'react'; +import { Card, cardDefaults } from '.'; +import { Icon } from '../Icon'; + +// Auto Docs +const meta = { + title: 'Components / Card', + component: Card, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'Used to render a card.', + }, + }, + + // Component-level argTypes + argTypes: { + title: { + description: 'The title of the card', + table: { + defaultValue: { summary: `${cardDefaults.title}` }, + }, + control: { + type: 'text', + }, + }, + subTitle: { + description: 'The subtitle of the card', + control: { + type: 'text', + }, + }, + icon: { + description: 'The icon on the card', + control: { + type: 'text', + }, + }, + iconAlignment: { + description: 'Whether the alignment of icon is horizontal or vertical', + table: { + defaultValue: { summary: `${cardDefaults.iconAlignment}` }, + }, + control: { + type: 'select', + }, + }, + percent: { + description: 'The percent value on the pill of the card', + control: { + type: 'number', + }, + }, + button: { + description: 'The button on the card', + control: { + type: 'text', + }, + }, + width: { + description: 'The width of the card', + control: { + type: 'text', + }, + }, + onClick: { + description: 'The on click function for the card', + }, + }, + + // Define default args + args: { + title: 'Title', + subTitle: 'Subtitle', + iconAlignment: 'horizontal', + width: '150px', + }, +} satisfies Meta<typeof Card>; + +export default meta; + +// Stories + +type Story = StoryObj<typeof meta>; + +// Basic story is what is displayed 1st in storybook +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => <Card {...props} />, +}; + +export const withChildren = () => ( + <Card title="Title" subTitle="Subtitle"> + <div style={{ backgroundColor: colors.gray[1000], padding: '8px 32px' }}>Children of the card (Swap me)</div> + </Card> +); + +export const withoutSubtitle = () => ( + <Card title="Title"> + <div style={{ backgroundColor: colors.gray[1000], padding: '8px 32px' }}>Children of the card (Swap me)</div> + </Card> +); + +export const withIcon = () => ( + <GridList> + <Card title="Title" subTitle="Subtitle" icon={<Icon icon="Cloud" color="gray" />} /> + <Card title="Title" subTitle="Subtitle" icon={<Icon icon="Cloud" color="gray" />} iconAlignment="vertical" /> + </GridList> +); + +export const withButton = () => ( + <Card + title="Title" + subTitle="Subtitle" + button={<Icon icon="Download" color="gray" size="2xl" />} + onClick={() => window.alert('Card clicked')} + /> +); + +export const withPercentPill = () => <Card title="Title" subTitle="Subtitle" percent={2} />; + +export const withAllTheElements = () => ( + <Card + title="Title" + subTitle="Subtitle" + percent={2} + icon={<Icon icon="Cloud" color="gray" />} + button={<Icon icon="Download" color="gray" size="2xl" />} + onClick={() => window.alert('Card clicked')} + > + <div style={{ backgroundColor: colors.gray[1000], padding: '8px 32px' }}>Children of the card (Swap me)</div> + </Card> +); diff --git a/datahub-web-react/src/alchemy-components/components/Card/Card.tsx b/datahub-web-react/src/alchemy-components/components/Card/Card.tsx new file mode 100644 index 00000000000000..55c581251bea99 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Card/Card.tsx @@ -0,0 +1,48 @@ +import React from 'react'; +import { CardProps } from './types'; +import { CardContainer, Header, SubTitle, SubTitleContainer, Title, TitleContainer } from './components'; +import { Pill } from '../Pills'; + +export const cardDefaults: CardProps = { + title: 'Title', + iconAlignment: 'horizontal', +}; + +export const Card = ({ + title = cardDefaults.title, + iconAlignment = cardDefaults.iconAlignment, + subTitle, + percent, + button, + onClick, + icon, + children, + width, +}: CardProps) => { + return ( + <CardContainer hasButton={!!button} onClick={onClick} width={width}> + <Header iconAlignment={iconAlignment}> + {icon && <div>{icon}</div>} + <TitleContainer> + <Title> + {title} + {!!percent && ( + <Pill + label={`${Math.abs(percent)}%`} + size="sm" + colorScheme={percent < 0 ? 'red' : 'green'} + leftIcon={percent < 0 ? 'TrendingDown' : 'TrendingUp'} + clickable={false} + /> + )} + + + {subTitle} + {button} + + + + {children} + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Card/components.ts b/datahub-web-react/src/alchemy-components/components/Card/components.ts new file mode 100644 index 00000000000000..bb3821fffc7f58 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Card/components.ts @@ -0,0 +1,59 @@ +import { colors, radius, spacing, typography } from '@src/alchemy-components/theme'; +import { IconAlignmentOptions } from '@src/alchemy-components/theme/config'; +import styled from 'styled-components'; + +export const CardContainer = styled.div<{ hasButton: boolean; width?: string }>(({ hasButton, width }) => ({ + border: `1px solid ${colors.gray[100]}`, + borderRadius: radius.lg, + padding: spacing.md, + minWidth: '150px', + boxShadow: '0px 1px 2px 0px rgba(33, 23, 95, 0.07)', + backgroundColor: colors.white, + display: 'flex', + flexDirection: 'column', + gap: spacing.md, + width, + + '&:hover': hasButton + ? { + border: `1px solid ${colors.violet[500]}`, + cursor: 'pointer', + } + : {}, +})); + +export const Header = styled.div<{ iconAlignment?: IconAlignmentOptions }>(({ iconAlignment }) => ({ + display: 'flex', + flexDirection: iconAlignment === 'horizontal' ? 'row' : 'column', + alignItems: iconAlignment === 'horizontal' ? 'center' : 'start', + gap: spacing.sm, + width: '100%', +})); + +export const TitleContainer = styled.div({ + display: 'flex', + flexDirection: 'column', + gap: 2, + width: '100%', +}); + +export const Title = styled.div({ + fontSize: typography.fontSizes.lg, + fontWeight: typography.fontWeights.bold, + color: colors.gray[600], + display: 'flex', + alignItems: 'center', + gap: spacing.xsm, +}); + +export const SubTitleContainer = styled.div({ + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', +}); + +export const SubTitle = styled.div({ + fontSize: typography.fontSizes.md, + fontWeight: typography.fontWeights.normal, + color: colors.gray[1700], +}); diff --git a/datahub-web-react/src/alchemy-components/components/Card/index.ts b/datahub-web-react/src/alchemy-components/components/Card/index.ts new file mode 100644 index 00000000000000..b0eed059aafd8a --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Card/index.ts @@ -0,0 +1,2 @@ +export { Card, cardDefaults } from './Card'; +export type { CardProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Card/types.ts b/datahub-web-react/src/alchemy-components/components/Card/types.ts new file mode 100644 index 00000000000000..e5b0e36f83e4ce --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Card/types.ts @@ -0,0 +1,13 @@ +import { IconAlignmentOptions } from '@src/alchemy-components/theme/config'; + +export interface CardProps { + title: string; + subTitle?: string; + percent?: number; + button?: React.ReactNode; + onClick?: () => void; + icon?: React.ReactNode; + iconAlignment?: IconAlignmentOptions; + children?: React.ReactNode; + width?: string; +} diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.stories.tsx b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.stories.tsx new file mode 100644 index 00000000000000..e546c2ea526cbc --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.stories.tsx @@ -0,0 +1,156 @@ +import React from 'react'; +import type { Meta, StoryObj } from '@storybook/react'; +import { GridList } from '@components/.docs/mdx-components'; +import { Checkbox, checkboxDefaults, CheckboxGroup } from './Checkbox'; +import { CheckboxProps } from './types'; +import { Heading } from '../Heading'; + +const MOCK_CHECKBOXES: CheckboxProps[] = [ + { + label: 'Label 1', + error: '', + isChecked: false, + isDisabled: false, + isIntermediate: false, + isRequired: false, + }, + { + label: 'Label 2', + error: '', + isChecked: false, + isDisabled: false, + isIntermediate: false, + isRequired: false, + }, + { + label: 'Label 3', + error: '', + isChecked: false, + isDisabled: false, + isIntermediate: false, + isRequired: false, + }, +]; + +const meta = { + title: 'Forms / Checkbox', + component: Checkbox, + parameters: { + layout: 'centered', + docs: { + subtitle: 'A component that is used to get user input in the state of a check box.', + }, + }, + argTypes: { + label: { + description: 'Label for the Checkbox.', + table: { + defaultValue: { summary: checkboxDefaults.label }, + }, + control: { + type: 'text', + }, + }, + error: { + description: 'Enforce error state on the Checkbox.', + table: { + defaultValue: { summary: checkboxDefaults.error }, + }, + control: { + type: 'text', + }, + }, + isChecked: { + description: 'Whether the Checkbox is checked.', + table: { + defaultValue: { summary: checkboxDefaults?.isChecked?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isDisabled: { + description: 'Whether the Checkbox is in disabled state.', + table: { + defaultValue: { summary: checkboxDefaults?.isDisabled?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isIntermediate: { + description: 'Whether the Checkbox is in intermediate state.', + table: { + defaultValue: { summary: checkboxDefaults?.isIntermediate?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isRequired: { + description: 'Whether the Checkbox is a required field.', + table: { + defaultValue: { summary: checkboxDefaults?.isRequired?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + }, + args: { + label: checkboxDefaults.label, + error: checkboxDefaults.error, + isChecked: checkboxDefaults.isChecked, + isDisabled: checkboxDefaults.isDisabled, + isIntermediate: checkboxDefaults.isIntermediate, + isRequired: checkboxDefaults.isRequired, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const states = () => ( + + + + + + +); + +export const intermediate = () => { + return ( + + + + + ); +}; + +export const disabledStates = () => ( + + + + + +); + +export const checkboxGroups = () => ( + +
+ Horizontal Checkbox Group + +
+
+ Vertical Checkbox Group + +
+ +); diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.tsx b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.tsx new file mode 100644 index 00000000000000..6ab4db74610e49 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Checkbox/Checkbox.tsx @@ -0,0 +1,103 @@ +import React, { useEffect, useState } from 'react'; +import { CheckboxProps, CheckboxGroupProps } from './types'; +import { + CheckboxBase, + CheckboxContainer, + CheckboxGroupContainer, + Checkmark, + HoverState, + Label, + Required, + StyledCheckbox, +} from './components'; + +export const checkboxDefaults: CheckboxProps = { + label: 'Label', + error: '', + isChecked: false, + isDisabled: false, + isIntermediate: false, + isRequired: false, + setIsChecked: () => {}, +}; + +export const Checkbox = ({ + label = checkboxDefaults.label, + error = checkboxDefaults.error, + isChecked = checkboxDefaults.isChecked, + isDisabled = checkboxDefaults.isDisabled, + isIntermediate = checkboxDefaults.isIntermediate, + isRequired = checkboxDefaults.isRequired, + setIsChecked = checkboxDefaults.setIsChecked, + ...props +}: CheckboxProps) => { + const [checked, setChecked] = useState(isChecked || false); + const [isHovering, setIsHovering] = useState(false); + + useEffect(() => { + setChecked(isChecked || false); + }, [isChecked]); + + const id = props.id || `checkbox-${label}`; + + return ( + + + { + if (!isDisabled) { + setChecked(!checked); + setIsChecked?.(!checked); + } + }} + > + null} + aria-labelledby={id} + aria-checked={checked} + {...props} + /> + setIsHovering(true)} + onMouseLeave={() => setIsHovering(false)} + /> + + + + ); +}; + +export const CheckboxGroup = ({ isVertical, checkboxes }: CheckboxGroupProps) => { + if (!checkboxes.length) { + return <>; + } + + return ( + + {checkboxes.map((checkbox) => { + const props = { ...checkbox }; + return ( + + + + ); + })} + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/components.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/components.ts new file mode 100644 index 00000000000000..6a4ad08c9c4ce6 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Checkbox/components.ts @@ -0,0 +1,91 @@ +import { borders, colors, spacing, transform, zIndices, radius } from '@components/theme'; +import styled from 'styled-components'; +import { getCheckboxColor, getCheckboxHoverBackgroundColor } from './utils'; +import { formLabelTextStyles } from '../commonStyles'; + +export const CheckboxContainer = styled.div({ + display: 'flex', + justifyContent: 'center', + alignItems: 'center', +}); + +export const Label = styled.div({ + ...formLabelTextStyles, +}); + +export const Required = styled.span({ + color: colors.red[500], + marginLeft: spacing.xxsm, +}); + +export const CheckboxBase = styled.div({ + position: 'relative', + width: '30px', + height: '30px', +}); + +export const StyledCheckbox = styled.input<{ + checked: boolean; + error: string; + disabled: boolean; +}>(({ error, checked, disabled }) => ({ + position: 'absolute', + opacity: 0, + height: 0, + width: 0, + '&:checked + div': { + backgroundColor: getCheckboxColor(checked, error, disabled, 'background'), + }, + '&:checked + div:after': { + display: 'block', + }, +})); + +export const Checkmark = styled.div<{ intermediate?: boolean; error: string; checked: boolean; disabled: boolean }>( + ({ intermediate, checked, error, disabled }) => ({ + position: 'absolute', + top: '4px', + left: '11px', + zIndex: zIndices.docked, + height: '18px', + width: '18px', + borderRadius: '3px', + border: `${borders['2px']} ${getCheckboxColor(checked, error, disabled, undefined)}`, + transition: 'all 0.2s ease-in-out', + cursor: 'pointer', + '&:after': { + content: '""', + position: 'absolute', + display: 'none', + left: !intermediate ? '6px' : '8px', + top: !intermediate ? '1px' : '3px', + width: !intermediate ? '5px' : '0px', + height: '10px', + border: 'solid white', + borderWidth: '0 3px 3px 0', + transform: !intermediate ? 'rotate(45deg)' : transform.rotate[90], + }, + }), +); + +export const HoverState = styled.div<{ isHovering: boolean; error: string; checked: boolean; disabled: boolean }>( + ({ isHovering, error, checked }) => ({ + width: '40px', + height: '40px', + backgroundColor: !isHovering ? 'transparent' : getCheckboxHoverBackgroundColor(checked, error), + position: 'absolute', + borderRadius: radius.full, + top: '-5px', + left: '2px', + zIndex: zIndices.hide, + }), +); + +export const CheckboxGroupContainer = styled.div<{ isVertical?: boolean }>(({ isVertical }) => ({ + display: 'flex', + flexDirection: isVertical ? 'column' : 'row', + justifyContent: 'center', + alignItems: 'center', + gap: spacing.md, + margin: spacing.xxsm, +})); diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/index.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/index.ts new file mode 100644 index 00000000000000..57e3d6d27856a5 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Checkbox/index.ts @@ -0,0 +1,2 @@ +export { Checkbox, CheckboxGroup, checkboxDefaults } from './Checkbox'; +export type { CheckboxProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/types.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/types.ts new file mode 100644 index 00000000000000..7ee10011689397 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Checkbox/types.ts @@ -0,0 +1,16 @@ +import { InputHTMLAttributes } from 'react'; + +export interface CheckboxProps extends InputHTMLAttributes { + label: string; + error?: string; + isChecked?: boolean; + setIsChecked?: React.Dispatch>; + isDisabled?: boolean; + isIntermediate?: boolean; + isRequired?: boolean; +} + +export interface CheckboxGroupProps { + isVertical?: boolean; + checkboxes: CheckboxProps[]; +} diff --git a/datahub-web-react/src/alchemy-components/components/Checkbox/utils.ts b/datahub-web-react/src/alchemy-components/components/Checkbox/utils.ts new file mode 100644 index 00000000000000..edf5d24596e1b4 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Checkbox/utils.ts @@ -0,0 +1,27 @@ +import theme, { colors } from '@components/theme'; + +const checkboxBackgroundDefault = { + default: colors.white, + checked: theme.semanticTokens.colors.primary, + error: theme.semanticTokens.colors.error, + disabled: colors.gray[300], +}; + +const checkboxHoverColors = { + default: colors.gray[100], + error: colors.red[100], + checked: colors.violet[100], +}; + +export function getCheckboxColor(checked: boolean, error: string, disabled: boolean, mode: 'background' | undefined) { + if (disabled) return checkboxBackgroundDefault.disabled; + if (error) return checkboxBackgroundDefault.error; + if (checked) return checkboxBackgroundDefault.checked; + return mode === 'background' ? checkboxBackgroundDefault.default : colors.gray[500]; +} + +export function getCheckboxHoverBackgroundColor(checked: boolean, error: string) { + if (error) return checkboxHoverColors.error; + if (checked) return checkboxHoverColors.checked; + return checkboxHoverColors.default; +} diff --git a/datahub-web-react/src/alchemy-components/components/Heading/Heading.stories.tsx b/datahub-web-react/src/alchemy-components/components/Heading/Heading.stories.tsx new file mode 100644 index 00000000000000..b8bd9f6420c006 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Heading/Heading.stories.tsx @@ -0,0 +1,98 @@ +import React from 'react'; + +import type { Meta, StoryObj, StoryFn } from '@storybook/react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; + +import { VerticalFlexGrid } from '@components/.docs/mdx-components'; +import { Heading, headingDefaults } from '.'; + +// Auto Docs +const meta = { + title: 'Typography / Heading', + component: Heading, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'Used to render semantic HTML heading elements.', + }, + }, + + // Component-level argTypes + argTypes: { + children: { + description: 'The content to display within the heading.', + table: { + type: { summary: 'string' }, + }, + control: { + type: 'text', + }, + }, + type: { + description: 'The type of heading to display.', + table: { + defaultValue: { summary: headingDefaults.type }, + }, + }, + size: { + description: 'Override the size of the heading.', + table: { + defaultValue: { summary: `${headingDefaults.size}` }, + }, + }, + color: { + description: 'Override the color of the heading.', + table: { + defaultValue: { summary: headingDefaults.color }, + }, + }, + weight: { + description: 'Override the weight of the heading.', + table: { + defaultValue: { summary: `${headingDefaults.weight}` }, + }, + }, + }, + + // Define defaults + args: { + children: 'The content to display within the heading.', + type: headingDefaults.type, + size: headingDefaults.size, + color: headingDefaults.color, + weight: headingDefaults.weight, + }, +} satisfies Meta; + +export default meta; + +// Stories + +type Story = StoryObj; + +// Basic story is what is displayed 1st in storybook & is used as the code sandbox +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => {props.children}, +}; + +export const sizes: StoryFn = (props: any) => ( + + H1 {props.children} + H2 {props.children} + H3 {props.children} + H4 {props.children} + H5 {props.children} + H6 {props.children} + +); + +export const withLink = () => ( + + The content to display within the heading + +); diff --git a/datahub-web-react/src/alchemy-components/components/Heading/Heading.tsx b/datahub-web-react/src/alchemy-components/components/Heading/Heading.tsx new file mode 100644 index 00000000000000..6449ff512adacc --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Heading/Heading.tsx @@ -0,0 +1,38 @@ +import React from 'react'; + +import { HeadingProps } from './types'; +import { H1, H2, H3, H4, H5, H6 } from './components'; + +export const headingDefaults: HeadingProps = { + type: 'h1', + color: 'inherit', + size: '2xl', + weight: 'medium', +}; + +export const Heading = ({ + type = headingDefaults.type, + size = headingDefaults.size, + color = headingDefaults.color, + weight = headingDefaults.weight, + children, +}: HeadingProps) => { + const sharedProps = { size, color, weight }; + + switch (type) { + case 'h1': + return

{children}

; + case 'h2': + return

{children}

; + case 'h3': + return

{children}

; + case 'h4': + return

{children}

; + case 'h5': + return
{children}
; + case 'h6': + return
{children}
; + default: + return

{children}

; + } +}; diff --git a/datahub-web-react/src/alchemy-components/components/Heading/components.ts b/datahub-web-react/src/alchemy-components/components/Heading/components.ts new file mode 100644 index 00000000000000..beea5338585d83 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Heading/components.ts @@ -0,0 +1,70 @@ +import styled from 'styled-components'; + +import { typography, colors } from '@components/theme'; +import { getColor, getFontSize } from '@components/theme/utils'; +import { HeadingProps } from './types'; + +const headingStyles = { + H1: { + fontSize: typography.fontSizes['4xl'], + lineHeight: typography.lineHeights['2xl'], + }, + H2: { + fontSize: typography.fontSizes['3xl'], + lineHeight: typography.lineHeights.xl, + }, + H3: { + fontSize: typography.fontSizes['2xl'], + lineHeight: typography.lineHeights.lg, + }, + H4: { + fontSize: typography.fontSizes.xl, + lineHeight: typography.lineHeights.lg, + }, + H5: { + fontSize: typography.fontSizes.lg, + lineHeight: typography.lineHeights.md, + }, + H6: { + fontSize: typography.fontSizes.md, + lineHeight: typography.lineHeights.xs, + }, +}; + +// Default styles +const baseStyles = { + fontFamily: typography.fonts.heading, + margin: 0, + + '& a': { + color: colors.violet[400], + textDecoration: 'none', + transition: 'color 0.15s ease', + + '&:hover': { + color: colors.violet[500], + }, + }, +}; + +// Prop Driven Styles +const propStyles = (props, isText = false) => { + const styles = {} as any; + if (props.size) styles.fontSize = getFontSize(props.size); + if (props.color) styles.color = getColor(props.color); + if (props.weight) styles.fontWeight = typography.fontWeights[props.weight]; + if (isText) styles.lineHeight = typography.lineHeights[props.size]; + return styles; +}; + +// Generate Headings +const headings = {} as any; + +['H1', 'H2', 'H3', 'H4', 'H5', 'H6'].forEach((heading) => { + const component = styled[heading.toLowerCase()]; + headings[heading] = component({ ...baseStyles, ...headingStyles[heading] }, (props: HeadingProps) => ({ + ...propStyles(props as HeadingProps), + })); +}); + +export const { H1, H2, H3, H4, H5, H6 } = headings; diff --git a/datahub-web-react/src/alchemy-components/components/Heading/index.ts b/datahub-web-react/src/alchemy-components/components/Heading/index.ts new file mode 100644 index 00000000000000..c414de6cc92f79 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Heading/index.ts @@ -0,0 +1,2 @@ +export { Heading, headingDefaults } from './Heading'; +export type { HeadingProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Heading/types.ts b/datahub-web-react/src/alchemy-components/components/Heading/types.ts new file mode 100644 index 00000000000000..96fcf1ea292bf7 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Heading/types.ts @@ -0,0 +1,9 @@ +import { HTMLAttributes } from 'react'; +import type { FontSizeOptions, FontColorOptions, FontWeightOptions } from '@components/theme/config'; + +export interface HeadingProps extends HTMLAttributes { + type?: 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6'; + size?: FontSizeOptions; + color?: FontColorOptions; + weight?: FontWeightOptions; +} diff --git a/datahub-web-react/src/alchemy-components/components/Icon/Icon.stories.tsx b/datahub-web-react/src/alchemy-components/components/Icon/Icon.stories.tsx new file mode 100644 index 00000000000000..3dcbd74ceb0b71 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Icon/Icon.stories.tsx @@ -0,0 +1,131 @@ +import React from 'react'; + +import type { Meta, StoryObj } from '@storybook/react'; + +import { GridList } from '@components/.docs/mdx-components'; +import { Icon, iconDefaults, AVAILABLE_ICONS } from '.'; + +// Auto Docs +const meta = { + title: 'Media / Icon', + component: Icon, + + // Display Properties + parameters: { + layout: 'centered', + badges: ['productionReady'], + docs: { + subtitle: 'A singular component for rendering the icons used throughout the application.', + description: { + component: '👉 See the [Icons Gallery](/docs/icons--docs) for more information.', + }, + }, + }, + + // Component-level argTypes + argTypes: { + icon: { + description: `The name of the icon to display.`, + type: 'string', + options: AVAILABLE_ICONS, + table: { + defaultValue: { summary: 'undefined' }, + }, + control: { + type: 'select', + }, + }, + variant: { + description: 'The variant of the icon to display.', + defaultValue: 'outline', + options: ['outline', 'filled'], + table: { + defaultValue: { summary: iconDefaults.variant }, + }, + }, + size: { + description: 'The size of the icon to display.', + defaultValue: 'lg', + table: { + defaultValue: { summary: iconDefaults.size }, + }, + }, + color: { + description: 'The color of the icon to display.', + options: ['inherit', 'white', 'black', 'violet', 'green', 'red', 'blue', 'gray'], + type: 'string', + table: { + defaultValue: { summary: iconDefaults.color }, + }, + control: { + type: 'select', + }, + }, + rotate: { + description: 'The rotation of the icon. Applies a CSS transformation.', + table: { + defaultValue: { summary: iconDefaults.rotate }, + }, + }, + }, + + // Define defaults for required args + args: { + icon: iconDefaults.icon, + }, +} satisfies Meta; + +export default meta; + +// Stories + +type Story = StoryObj; + +// Basic story is what is displayed 1st in storybook +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const filled = () => ( + + + + + +); + +export const sizes = () => ( + + + + + + + + + + +); + +export const colors = () => ( + + + + + + + + + +); + +export const rotation = () => ( + + + + + + +); diff --git a/datahub-web-react/src/alchemy-components/components/Icon/Icon.tsx b/datahub-web-react/src/alchemy-components/components/Icon/Icon.tsx new file mode 100644 index 00000000000000..50c30d7203aed8 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Icon/Icon.tsx @@ -0,0 +1,59 @@ +import React from 'react'; + +import { getFontSize, getColor, getRotationTransform } from '@components/theme/utils'; + +import { IconProps } from './types'; +import { IconWrapper } from './components'; +import { getIconNames, getIconComponent } from './utils'; + +export const iconDefaults: IconProps = { + icon: 'AccountCircle', + variant: 'outline', + size: '4xl', + color: 'inherit', + rotate: '0', +}; + +export const Icon = ({ + icon, + variant = iconDefaults.variant, + size = iconDefaults.size, + color = iconDefaults.color, + rotate = iconDefaults.rotate, + ...props +}: IconProps) => { + const { filled, outlined } = getIconNames(); + + // Return early if no icon is provided + if (!icon) return null; + + // Get outlined icon component name + const isOutlined = variant === 'outline'; + const outlinedIconName = `${icon}Outlined`; + + // Warn if the icon does not have the specified variant + if (variant === 'outline' && !outlined.includes(outlinedIconName)) { + console.warn(`Icon "${icon}" does not have an outlined variant.`); + return null; + } + + // Warn if the icon does not have the specified variant + if (variant === 'filled' && !filled.includes(icon)) { + console.warn(`Icon "${icon}" does not have a filled variant.`); + return null; + } + + // Get outlined icon component + const IconComponent = getIconComponent(isOutlined ? outlinedIconName : icon); + + return ( + + + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Icon/components.ts b/datahub-web-react/src/alchemy-components/components/Icon/components.ts new file mode 100644 index 00000000000000..82e9c9a8fcae00 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Icon/components.ts @@ -0,0 +1,19 @@ +import styled from 'styled-components'; + +export const IconWrapper = styled.div<{ size: string; rotate?: string }>` + position: relative; + + display: flex; + align-items: center; + justify-content: center; + + width: ${({ size }) => size}; + height: ${({ size }) => size}; + + & svg { + width: 100%; + height: 100%; + + transform: ${({ rotate }) => rotate}; + } +`; diff --git a/datahub-web-react/src/alchemy-components/components/Icon/constants.ts b/datahub-web-react/src/alchemy-components/components/Icon/constants.ts new file mode 100644 index 00000000000000..25145a5970f0f2 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Icon/constants.ts @@ -0,0 +1,547 @@ +export const AVAILABLE_ICONS = [ + 'AccountCircle', + 'AccountTree', + 'AddCircle', + 'AddLink', + 'Add', + 'AddTask', + 'AddToPhotos', + 'Adjust', + 'AllInclusive', + 'Analytics', + 'Anchor', + 'Animation', + 'Announcement', + 'Api', + 'Approval', + 'Archive', + 'ArrowBack', + 'ArrowCircleDown', + 'ArrowCircleLeft', + 'ArrowCircleRight', + 'ArrowCircleUp', + 'ArrowDownward', + 'ArrowForward', + 'ArrowOutward', + 'ArrowUpward', + 'ArtTrack', + 'Article', + 'Assistant', + 'AttachFile', + 'Attachment', + 'AutoAwesome', + 'AutoFixHigh', + 'AutoFixOff', + 'AutoGraph', + 'AutoMode', + 'AutoStories', + 'AvTimer', + 'Backspace', + 'Backup', + 'BackupTable', + 'Badge', + 'Balance', + 'BarChart', + 'BatchPrediction', + 'Block', + 'Bolt', + 'Book', + 'BookmarkAdd', + 'BookmarkAdded', + 'BookmarkBorder', + 'Bookmark', + 'BookmarkRemove', + 'Bookmarks', + 'Brush', + 'BubbleChart', + 'BugReport', + 'BuildCircle', + 'Build', + 'BusinessCenter', + 'Business', + 'Cable', + 'Cached', + 'Calculate', + 'CalendarMonth', + 'CalendarToday', + 'CalendarViewDay', + 'Campaign', + 'Cancel', + 'CandlestickChart', + 'CardGiftcard', + 'CardMembership', + 'Cases', + 'Cast', + 'Category', + 'Celebration', + 'CellTower', + 'ChangeHistory', + 'ChatBubble', + 'Chat', + 'CheckBox', + 'CheckCircle', + 'Check', + 'Checklist', + 'ChevronLeft', + 'ChevronRight', + 'Class', + 'CloseFullscreen', + 'Close', + 'CloudCircle', + 'CloudDone', + 'CloudDownload', + 'CloudOff', + 'Cloud', + 'CloudQueue', + 'CloudSync', + 'CloudUpload', + 'CoPresent', + 'CodeOff', + 'Code', + 'ColorLens', + 'Colorize', + 'CommentBank', + 'Comment', + 'CommentsDisabled', + 'Commit', + 'CompareArrows', + 'Compare', + 'Compress', + 'Computer', + 'Construction', + 'ContactPage', + 'ContactSupport', + 'Contacts', + 'ContentCopy', + 'ContentCut', + 'Contrast', + 'ControlPoint', + 'Cookie', + 'CopyAll', + 'Copyright', + 'CorporateFare', + 'Cottage', + 'CreateNewFolder', + 'CrisisAlert', + 'Cyclone', + 'Dangerous', + 'DarkMode', + 'DashboardCustomize', + 'Dashboard', + 'DataArray', + 'DataObject', + 'DataThresholding', + 'DataUsage', + 'DatasetLinked', + 'Dataset', + 'DateRange', + 'DeleteForever', + 'Delete', + 'DeleteSweep', + 'Description', + 'Deselect', + 'DesignServices', + 'Details', + 'DeviceHub', + 'DeviceThermostat', + 'Diamond', + 'Difference', + 'DisabledByDefault', + 'DiscFull', + 'Discount', + 'DisplaySettings', + 'Diversity2', + 'Dns', + 'DoNotDisturb', + 'DocumentScanner', + 'DomainAdd', + 'DomainDisabled', + 'Domain', + 'DomainVerification', + 'DoneAll', + 'DonutLarge', + 'DonutSmall', + 'DoubleArrow', + 'DownloadDone', + 'DownloadForOffline', + 'Download', + 'Downloading', + 'Drafts', + 'DragHandle', + 'DragIndicator', + 'Draw', + 'DriveFileMove', + 'DriveFolderUpload', + 'DynamicFeed', + 'DynamicForm', + 'EditCalendar', + 'EditLocation', + 'EditNote', + 'EditOff', + 'Edit', + 'Eject', + 'ElectricBolt', + 'EmergencyShare', + 'EnhancedEncryption', + 'Equalizer', + 'Error', + 'EventAvailable', + 'EventBusy', + 'EventNote', + 'Event', + 'EventRepeat', + 'ExitToApp', + 'Expand', + 'ExploreOff', + 'Explore', + 'Exposure', + 'ExtensionOff', + 'Extension', + 'FastForward', + 'FastRewind', + 'FavoriteBorder', + 'Favorite', + 'FeaturedPlayList', + 'Feed', + 'Feedback', + 'FileCopy', + 'FileDownloadOff', + 'FileDownload', + 'FileOpen', + 'FilePresent', + 'FileUpload', + 'FilterAltOff', + 'FilterAlt', + 'FilterListOff', + 'FindInPage', + 'FindReplace', + 'FirstPage', + 'FitScreen', + 'FlagCircle', + 'Flag', + 'Flaky', + 'Flare', + 'FlashOff', + 'FlashOn', + 'FlightLand', + 'Flight', + 'FlightTakeoff', + 'FmdBad', + 'FmdGood', + 'FolderCopy', + 'FolderDelete', + 'FolderOff', + 'FolderOpen', + 'Folder', + 'FolderShared', + 'FolderSpecial', + 'FolderZip', + 'ForkLeft', + 'ForkRight', + 'FormatListBulleted', + 'FormatListNumbered', + 'Forum', + 'FullscreenExit', + 'Fullscreen', + 'Functions', + 'GetApp', + 'GppBad', + 'GppGood', + 'GppMaybe', + 'GpsFixed', + 'GpsNotFixed', + 'GpsOff', + 'Grading', + 'Grain', + 'GraphicEq', + 'Grid3x3', + 'Grid4x4', + 'GridGoldenratio', + 'GridOff', + 'GridOn', + 'GridView', + 'GroupAdd', + 'Group', + 'GroupRemove', + 'GroupWork', + 'Groups', + 'Handshake', + 'Handyman', + 'Hardware', + 'HealthAndSafety', + 'HelpCenter', + 'Help', + 'Hexagon', + 'HideSource', + 'Highlight', + 'History', + 'HistoryToggleOff', + 'Home', + 'Hub', + 'Image', + 'ImageSearch', + 'Inbox', + 'Info', + 'Input', + 'InsertChart', + 'InsertComment', + 'InsertDriveFile', + 'Insights', + 'Interests', + 'Inventory2', + 'Inventory', + 'KeyOff', + 'Key', + 'LabelImportant', + 'LabelOff', + 'Label', + 'Lan', + 'Landscape', + 'Language', + 'LastPage', + 'Launch', + 'LayersClear', + 'Layers', + 'Leaderboard', + 'LegendToggle', + 'LibraryAddCheck', + 'LibraryAdd', + 'LightMode', + 'Lightbulb', + 'LineAxis', + 'LineStyle', + 'LineWeight', + 'LinearScale', + 'LinkOff', + 'Link', + 'List', + 'LockOpen', + 'Lock', + 'LockReset', + 'Login', + 'Logout', + 'Loupe', + 'LowPriority', + 'Loyalty', + 'Mail', + 'ManageAccounts', + 'ManageHistory', + 'ManageSearch', + 'Map', + 'MapsUgc', + 'MarkAsUnread', + 'MeetingRoom', + 'Memory', + 'MenuBook', + 'MenuOpen', + 'Menu', + 'Merge', + 'MergeType', + 'Message', + 'MiscellaneousServices', + 'MoodBad', + 'Mood', + 'MoreHoriz', + 'MoreTime', + 'MoreVert', + 'MoveDown', + 'MoveToInbox', + 'MoveUp', + 'MultilineChart', + 'MultipleStop', + 'Nat', + 'NewReleases', + 'NightsStay', + 'NoAccounts', + 'NoEncryption', + 'NotStarted', + 'NoteAdd', + 'NotificationAdd', + 'NotificationImportant', + 'NotificationsActive', + 'NotificationsOff', + 'Notifications', + 'NotificationsPaused', + 'OpenInFull', + 'OpenInNew', + 'Outbound', + 'Outbox', + 'Output', + 'Pageview', + 'Password', + 'PauseCircle', + 'PendingActions', + 'Pending', + 'People', + 'PersonAddAlt1', + 'PersonOff', + 'Person', + 'PersonRemoveAlt1', + 'PersonSearch', + 'PinDrop', + 'PivotTableChart', + 'Place', + 'PlayArrow', + 'PlayCircle', + 'Policy', + 'Poll', + 'Polyline', + 'PostAdd', + 'Preview', + 'PrivacyTip', + 'PublicOff', + 'Public', + 'Publish', + 'PushPin', + 'QueryStats', + 'QuestionAnswer', + 'Queue', + 'Radar', + 'ReadMore', + 'Redo', + 'Refresh', + 'RemoveCircle', + 'Replay', + 'ReplyAll', + 'Reply', + 'Report', + 'ReportProblem', + 'Restore', + 'RocketLaunch', + 'Rocket', + 'Route', + 'RssFeed', + 'Rule', + 'RunningWithErrors', + 'SatelliteAlt', + 'SaveAlt', + 'Schedule', + 'Schema', + 'Science', + 'SearchOff', + 'Search', + 'Security', + 'Sell', + 'Sensors', + 'SentimentDissatisfied', + 'SentimentNeutral', + 'SentimentSatisfied', + 'Settings', + 'Share', + 'Shield', + 'ShortText', + 'Shortcut', + 'ShowChart', + 'Shuffle', + 'Signpost', + 'SkipNext', + 'SkipPrevious', + 'SortByAlpha', + 'Sort', + 'Source', + 'SpaceDashboard', + 'Speed', + 'SsidChart', + 'StackedBarChart', + 'StackedLineChart', + 'StarBorder', + 'StarHalf', + 'Star', + 'Start', + 'StickyNote2', + 'StopCircle', + 'Storage', + 'Storm', + 'Straight', + 'Stream', + 'Style', + 'SubdirectoryArrowLeft', + 'SubdirectoryArrowRight', + 'Subject', + 'Subscriptions', + 'SubtitlesOff', + 'Support', + 'SwapHoriz', + 'SwapVert', + 'SwitchAccount', + 'SwitchLeft', + 'SwitchRight', + 'SyncAlt', + 'SyncDisabled', + 'SyncLock', + 'Sync', + 'SyncProblem', + 'TableChart', + 'TableRows', + 'TableView', + 'Tag', + 'TaskAlt', + 'Terminal', + 'ThumbDown', + 'ThumbUp', + 'ThumbsUpDown', + 'Timelapse', + 'Timeline', + 'TipsAndUpdates', + 'Toc', + 'TrackChanges', + 'TrendingDown', + 'TrendingFlat', + 'TrendingUp', + 'Tune', + 'Tungsten', + 'TurnLeft', + 'TurnRight', + 'TurnSlightLeft', + 'TurnSlightRight', + 'Unarchive', + 'Undo', + 'UnfoldLessDouble', + 'UnfoldLess', + 'UnfoldMoreDouble', + 'UnfoldMore', + 'Unsubscribe', + 'Upcoming', + 'UpdateDisabled', + 'Update', + 'Upgrade', + 'UploadFile', + 'Upload', + 'Verified', + 'VerifiedUser', + 'ViewAgenda', + 'ViewArray', + 'ViewCarousel', + 'ViewColumn', + 'ViewComfy', + 'ViewCompact', + 'ViewCozy', + 'ViewDay', + 'ViewHeadline', + 'ViewKanban', + 'ViewList', + 'ViewModule', + 'ViewQuilt', + 'ViewSidebar', + 'ViewStream', + 'ViewTimeline', + 'ViewWeek', + 'VisibilityOff', + 'Visibility', + 'Warehouse', + 'Warning', + 'Webhook', + 'Whatshot', + 'Widgets', + 'Wifi', + 'Window', + 'WorkHistory', + 'WorkOff', + 'Work', + 'WorkspacePremium', + 'Workspaces', + 'Wysiwyg', + 'ZoomInMap', + 'ZoomIn', + 'ZoomOutMap', +]; diff --git a/datahub-web-react/src/alchemy-components/components/Icon/index.ts b/datahub-web-react/src/alchemy-components/components/Icon/index.ts new file mode 100644 index 00000000000000..23ca0a7ef7da2f --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Icon/index.ts @@ -0,0 +1,3 @@ +export { Icon, iconDefaults } from './Icon'; +export type { IconProps, IconNames } from './types'; +export { AVAILABLE_ICONS } from './constants'; diff --git a/datahub-web-react/src/alchemy-components/components/Icon/types.ts b/datahub-web-react/src/alchemy-components/components/Icon/types.ts new file mode 100644 index 00000000000000..f5a050e9338a71 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Icon/types.ts @@ -0,0 +1,23 @@ +import { HTMLAttributes } from 'react'; + +import type { FontSizeOptions, FontColorOptions, RotationOptions } from '@components/theme/config'; +import { AVAILABLE_ICONS } from './constants'; + +// Utility function to create an enum from an array of strings +function createEnum(values: T[]): { [K in T]: K } { + return values.reduce((acc, value) => { + acc[value] = value; + return acc; + }, Object.create(null)); +} + +const names = createEnum(AVAILABLE_ICONS); +export type IconNames = keyof typeof names; + +export interface IconProps extends HTMLAttributes { + icon: IconNames; + variant?: 'filled' | 'outline'; + size?: FontSizeOptions; + color?: FontColorOptions; + rotate?: RotationOptions; +} diff --git a/datahub-web-react/src/alchemy-components/components/Icon/utils.ts b/datahub-web-react/src/alchemy-components/components/Icon/utils.ts new file mode 100644 index 00000000000000..1137b3da28bc7a --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Icon/utils.ts @@ -0,0 +1,29 @@ +import * as materialIcons from '@mui/icons-material'; + +export const getIconNames = () => { + // We only want "Filled" (mui default) and "Outlined" icons + const filtered = Object.keys(materialIcons).filter( + (key) => + !key.includes('Filled') && !key.includes('TwoTone') && !key.includes('Rounded') && !key.includes('Sharp'), + ); + + const filled: string[] = []; + const outlined: string[] = []; + + filtered.forEach((key) => { + if (key.includes('Outlined')) { + outlined.push(key); + } else if (!key.includes('Outlined')) { + filled.push(key); + } + }); + + return { + filled, + outlined, + }; +}; + +export const getIconComponent = (icon: string) => { + return materialIcons[icon]; +}; diff --git a/datahub-web-react/src/alchemy-components/components/Input/Input.stories.tsx b/datahub-web-react/src/alchemy-components/components/Input/Input.stories.tsx new file mode 100644 index 00000000000000..053e952b62a2e9 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Input/Input.stories.tsx @@ -0,0 +1,177 @@ +import React from 'react'; + +import type { Meta, StoryObj } from '@storybook/react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; + +import { GridList } from '@components/.docs/mdx-components'; +import { AVAILABLE_ICONS } from '../Icon'; + +import { Input, inputDefaults } from './Input'; + +const meta = { + title: 'Forms / Input', + component: Input, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'A component that is used to get user input in a single line field.', + }, + }, + + // Component-level argTypes + argTypes: { + value: { + description: 'Value for the Input.', + table: { + defaultValue: { summary: inputDefaults.value as string }, + }, + control: { + type: 'text', + }, + }, + label: { + description: 'Label for the Input.', + table: { + defaultValue: { summary: inputDefaults.label }, + }, + control: { + type: 'text', + }, + }, + placeholder: { + description: 'Placeholder for the Input.', + table: { + defaultValue: { summary: inputDefaults.placeholder }, + }, + control: { + type: 'text', + }, + }, + icon: { + description: 'The icon to display in the Input.', + type: 'string', + options: AVAILABLE_ICONS, + table: { + defaultValue: { summary: 'undefined' }, + }, + control: { + type: 'select', + }, + }, + error: { + description: 'Enforce error state on the Input.', + table: { + defaultValue: { summary: inputDefaults.error }, + }, + control: { + type: 'text', + }, + }, + warning: { + description: 'Enforce warning state on the Input.', + table: { + defaultValue: { summary: inputDefaults.warning }, + }, + control: { + type: 'text', + }, + }, + isSuccess: { + description: 'Enforce success state on the Input.', + table: { + defaultValue: { summary: inputDefaults?.isSuccess?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isDisabled: { + description: 'Whether the Input is in disabled state.', + table: { + defaultValue: { summary: inputDefaults?.isDisabled?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isInvalid: { + description: 'Whether the Input is an invalid state.', + table: { + defaultValue: { summary: inputDefaults?.isInvalid?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isReadOnly: { + description: 'Whether the Input is in readonly mode.', + table: { + defaultValue: { summary: inputDefaults?.isReadOnly?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isPassword: { + description: 'Whether the Input has a password type.', + table: { + defaultValue: { summary: inputDefaults?.isPassword?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isRequired: { + description: 'Whether the Input is a required field.', + table: { + defaultValue: { summary: inputDefaults?.isRequired?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + }, + args: { + value: inputDefaults.value, + label: inputDefaults.label, + placeholder: inputDefaults.placeholder, + icon: inputDefaults.icon, + error: inputDefaults.error, + warning: inputDefaults.warning, + isSuccess: inputDefaults.isSuccess, + isDisabled: inputDefaults.isDisabled, + isInvalid: inputDefaults.isInvalid, + isReadOnly: inputDefaults.isReadOnly, + isPassword: inputDefaults.isPassword, + isRequired: inputDefaults.isRequired, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const status = () => ( + + + + + +); + +export const states = () => ( + + + + + + +); diff --git a/datahub-web-react/src/alchemy-components/components/Input/Input.tsx b/datahub-web-react/src/alchemy-components/components/Input/Input.tsx new file mode 100644 index 00000000000000..976fc47ffc5948 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Input/Input.tsx @@ -0,0 +1,97 @@ +import { Tooltip } from '@components'; +import React from 'react'; + +import { InputProps } from './types'; + +import { ErrorMessage, InputContainer, InputField, InputWrapper, Label, Required, WarningMessage } from './components'; + +import { Icon } from '../Icon'; +import { getInputType } from './utils'; + +export const inputDefaults: InputProps = { + value: '', + setValue: () => {}, + label: 'Label', + placeholder: 'Placeholder', + error: '', + warning: '', + isSuccess: false, + isDisabled: false, + isInvalid: false, + isReadOnly: false, + isPassword: false, + isRequired: false, + errorOnHover: false, + type: 'text', +}; + +export const Input = ({ + value = inputDefaults.value, + setValue = inputDefaults.setValue, + label = inputDefaults.label, + placeholder = inputDefaults.placeholder, + icon, // default undefined + error = inputDefaults.error, + warning = inputDefaults.warning, + isSuccess = inputDefaults.isSuccess, + isDisabled = inputDefaults.isDisabled, + isInvalid = inputDefaults.isInvalid, + isReadOnly = inputDefaults.isReadOnly, + isPassword = inputDefaults.isPassword, + isRequired = inputDefaults.isRequired, + errorOnHover = inputDefaults.errorOnHover, + type = inputDefaults.type, + id, + ...props +}: InputProps) => { + // Invalid state is always true if error is present + let invalid = isInvalid; + if (error) invalid = true; + + // Show/hide password text + const [showPassword, setShowPassword] = React.useState(false); + const passwordIcon = showPassword ? 'Visibility' : 'VisibilityOff'; + + // Input base props + const inputBaseProps = { + label, + isSuccess, + error, + warning, + isDisabled, + isInvalid: invalid, + }; + + return ( + + {label && ( + + )} + + {icon && } + setValue?.(e.target.value)} + type={getInputType(type, isPassword, showPassword)} + placeholder={placeholder} + readOnly={isReadOnly} + disabled={isDisabled} + required={isRequired} + id={id} + /> + {!isPassword && ( + + {invalid && } + {isSuccess && } + {warning && } + + )} + {isPassword && setShowPassword(!showPassword)} icon={passwordIcon} size="lg" />} + + {invalid && error && !errorOnHover && {error}} + {warning && {warning}} + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Input/components.ts b/datahub-web-react/src/alchemy-components/components/Input/components.ts new file mode 100644 index 00000000000000..d1c337642d9cd8 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Input/components.ts @@ -0,0 +1,92 @@ +import styled from 'styled-components'; + +import theme, { borders, colors, radius, spacing, typography } from '@components/theme'; +import { getStatusColors } from '@components/theme/utils'; + +import { + INPUT_MAX_HEIGHT, + formLabelTextStyles, + inputValueTextStyles, + inputPlaceholderTextStyles, +} from '../commonStyles'; + +import type { InputProps } from './types'; + +const defaultFlexStyles = { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', +}; + +const defaultMessageStyles = { + marginTop: spacing.xxsm, + fontSize: typography.fontSizes.sm, +}; + +export const InputWrapper = styled.div({ + ...defaultFlexStyles, + alignItems: 'flex-start', + flexDirection: 'column', + width: '100%', +}); + +export const InputContainer = styled.div( + ({ isSuccess, warning, isDisabled, isInvalid }: InputProps) => ({ + border: `${borders['1px']} ${getStatusColors(isSuccess, warning, isInvalid)}`, + backgroundColor: isDisabled ? colors.gray[100] : colors.white, + paddingRight: spacing.md, + }), + { + ...defaultFlexStyles, + width: '100%', + maxHeight: INPUT_MAX_HEIGHT, + overflow: 'hidden', + borderRadius: radius.md, + flex: 1, + color: colors.gray[400], // 1st icon color + + '&:focus-within': { + borderColor: colors.violet[200], + outline: `${borders['1px']} ${colors.violet[200]}`, + }, + }, +); + +export const InputField = styled.input({ + padding: `${spacing.sm} ${spacing.md}`, + lineHeight: typography.lineHeights.normal, + maxHeight: INPUT_MAX_HEIGHT, + border: borders.none, + width: '100%', + + // Shared common input text styles + ...inputValueTextStyles(), + + '&::placeholder': { + ...inputPlaceholderTextStyles, + }, + + '&:focus': { + outline: 'none', + }, +}); + +export const Required = styled.span({ + color: colors.red[500], +}); + +export const Label = styled.div({ + ...formLabelTextStyles, + marginBottom: spacing.xsm, + textAlign: 'left', +}); + +export const ErrorMessage = styled.div({ + ...defaultMessageStyles, + color: theme.semanticTokens.colors.error, +}); + +export const WarningMessage = styled.div({ + ...defaultMessageStyles, + color: theme.semanticTokens.colors.warning, +}); diff --git a/datahub-web-react/src/alchemy-components/components/Input/index.ts b/datahub-web-react/src/alchemy-components/components/Input/index.ts new file mode 100644 index 00000000000000..336a9b4dd08e97 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Input/index.ts @@ -0,0 +1,2 @@ +export { Input, inputDefaults } from './Input'; +export type { InputProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Input/types.ts b/datahub-web-react/src/alchemy-components/components/Input/types.ts new file mode 100644 index 00000000000000..1b2abf132d3283 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Input/types.ts @@ -0,0 +1,22 @@ +import { InputHTMLAttributes } from 'react'; + +import { IconNames } from '../Icon'; + +export interface InputProps extends InputHTMLAttributes { + value?: string | number | readonly string[] | undefined; + setValue?: React.Dispatch>; + label: string; + placeholder?: string; + icon?: IconNames; + error?: string; + warning?: string; + isSuccess?: boolean; + isDisabled?: boolean; + isInvalid?: boolean; + isReadOnly?: boolean; + isPassword?: boolean; + isRequired?: boolean; + errorOnHover?: boolean; + id?: string; + type?: string; +} diff --git a/datahub-web-react/src/alchemy-components/components/Input/utils.ts b/datahub-web-react/src/alchemy-components/components/Input/utils.ts new file mode 100644 index 00000000000000..142a93232485b3 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Input/utils.ts @@ -0,0 +1,5 @@ +export const getInputType = (type?: string, isPassword?: boolean, showPassword?: boolean) => { + if (type) return type; + if (isPassword && !showPassword) return 'password'; + return 'text'; +}; diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.stories.tsx b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.stories.tsx new file mode 100644 index 00000000000000..8cce0369918a2e --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.stories.tsx @@ -0,0 +1,96 @@ +import React from 'react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import type { Meta, StoryObj } from '@storybook/react'; +import { LineChart } from './LineChart'; +import { getMockedProps } from '../BarChart/utils'; + +const meta = { + title: 'Charts / LineChart', + component: LineChart, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.EXPERIMENTAL], + docs: { + subtitle: 'A component that is used to show LineChart', + }, + }, + + // Component-level argTypes + argTypes: { + data: { + description: 'Array of datum to show', + }, + xAccessor: { + description: 'A function to convert datum to value of X', + }, + yAccessor: { + description: 'A function to convert datum to value of Y', + }, + renderTooltipContent: { + description: 'A function to replace default rendering of toolbar', + }, + margin: { + description: 'Add margins to chart', + }, + leftAxisTickFormat: { + description: 'A function to format labels of left axis', + }, + leftAxisTickLabelProps: { + description: 'Props for label of left axis', + }, + bottomAxisTickFormat: { + description: 'A function to format labels of bottom axis', + }, + bottomAxisTickLabelProps: { + description: 'Props for label of bottom axis', + }, + lineColor: { + description: 'Color of line on chart', + control: { + type: 'color', + }, + }, + areaColor: { + description: 'Color of area under line', + control: { + type: 'color', + }, + }, + gridColor: { + description: "Color of grid's lines", + control: { + type: 'color', + }, + }, + renderGradients: { + description: 'A function to render different gradients that can be used as colors', + }, + toolbarVerticalCrosshairStyle: { + description: "Styles of toolbar's vertical line", + }, + renderTooltipGlyph: { + description: 'A function to render a glyph', + }, + }, + + // Define defaults + args: { + ...getMockedProps(), + renderTooltipContent: (datum) => <>DATUM: {JSON.stringify(datum)}, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => ( +
+ +
+ ), +}; diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.tsx b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.tsx new file mode 100644 index 00000000000000..22580122ccf84f --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/LineChart/LineChart.tsx @@ -0,0 +1,178 @@ +import { colors } from '@src/alchemy-components/theme'; +// import { abbreviateNumber } from '@src/app/dataviz/utils'; +import { TickLabelProps } from '@visx/axis'; +import { curveMonotoneX } from '@visx/curve'; +import { LinearGradient } from '@visx/gradient'; +import { ParentSize } from '@visx/responsive'; +import { AreaSeries, Axis, AxisScale, Grid, LineSeries, Tooltip, XYChart } from '@visx/xychart'; +import dayjs from 'dayjs'; +import React, { useState } from 'react'; +import { Popover } from '../Popover'; +import { ChartWrapper } from './components'; +import { LineChartProps } from './types'; +import { abbreviateNumber } from '../dataviz/utils'; + +const commonTickLabelProps: TickLabelProps = { + fontSize: 10, + fontFamily: 'Mulish', + fill: colors.gray[1700], +}; + +const GLYPH_DROP_SHADOW_FILTER = ` + drop-shadow(0px 1px 3px rgba(33, 23, 95, 0.30)) + drop-shadow(0px 2px 5px rgba(33, 23, 95, 0.25)) + drop-shadow(0px -2px 5px rgba(33, 23, 95, 0.25) +`; + +export const lineChartDefault: LineChartProps = { + data: [], + xAccessor: (datum) => datum?.x, + yAccessor: (datum) => datum?.y, + leftAxisTickFormat: abbreviateNumber, + leftAxisTickLabelProps: { + ...commonTickLabelProps, + textAnchor: 'end', + }, + bottomAxisTickFormat: (x) => dayjs(x).format('D MMM'), + bottomAxisTickLabelProps: { + ...commonTickLabelProps, + textAnchor: 'middle', + verticalAnchor: 'start', + }, + lineColor: colors.violet[500], + areaColor: 'url(#line-gradient)', + gridColor: '#e0e0e0', + renderGradients: () => ( + + ), + toolbarVerticalCrosshairStyle: { + stroke: colors.white, + strokeWidth: 2, + filter: GLYPH_DROP_SHADOW_FILTER, + }, + renderTooltipGlyph: (props) => { + return ( + <> + + + + ); + }, +}; + +export function LineChart({ + data, + xAccessor = lineChartDefault.xAccessor, + yAccessor = lineChartDefault.yAccessor, + renderTooltipContent, + margin, + leftAxisTickFormat = lineChartDefault.leftAxisTickFormat, + leftAxisTickLabelProps = lineChartDefault.leftAxisTickLabelProps, + bottomAxisTickFormat = lineChartDefault.bottomAxisTickFormat, + bottomAxisTickLabelProps = lineChartDefault.bottomAxisTickLabelProps, + lineColor = lineChartDefault.lineColor, + areaColor = lineChartDefault.areaColor, + gridColor = lineChartDefault.gridColor, + renderGradients = lineChartDefault.renderGradients, + toolbarVerticalCrosshairStyle = lineChartDefault.toolbarVerticalCrosshairStyle, + renderTooltipGlyph = lineChartDefault.renderTooltipGlyph, +}: LineChartProps) { + const [showGrid, setShowGrid] = useState(false); + + // FYI: additional margins to show left and bottom axises + const internalMargin = { + top: (margin?.top ?? 0) + 30, + right: (margin?.right ?? 0) + 20, + bottom: (margin?.bottom ?? 0) + 35, + left: (margin?.left ?? 0) + 40, + }; + + const accessors = { xAccessor, yAccessor }; + + return ( + setShowGrid(true)} onMouseLeave={() => setShowGrid(false)}> + + {({ width, height }) => { + return ( + + {renderGradients?.()} + + + + + + + + {showGrid && ( + + )} + + + dataKey="line-chart-seria-01" + data={data} + fill={areaColor} + curve={curveMonotoneX} + {...accessors} + /> + + dataKey="line-chart-seria-01" + data={data} + stroke={lineColor} + curve={curveMonotoneX} + {...accessors} + /> + + + snapTooltipToDatumX + snapTooltipToDatumY + showVerticalCrosshair + applyPositionStyle + showSeriesGlyphs + verticalCrosshairStyle={toolbarVerticalCrosshairStyle} + renderGlyph={renderTooltipGlyph} + unstyled + renderTooltip={({ tooltipData }) => { + return ( + tooltipData?.nearestDatum && ( + + ) + ); + }} + /> + + ); + }} + + + ); +} diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/components.tsx b/datahub-web-react/src/alchemy-components/components/LineChart/components.tsx new file mode 100644 index 00000000000000..fb6c0cf1ced784 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/LineChart/components.tsx @@ -0,0 +1,8 @@ +import styled from 'styled-components'; + +export const ChartWrapper = styled.div` + width: 100%; + height: 100%; + position: relative; + cursor: pointer; +`; diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/index.ts b/datahub-web-react/src/alchemy-components/components/LineChart/index.ts new file mode 100644 index 00000000000000..7fca9300d578ca --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/LineChart/index.ts @@ -0,0 +1 @@ +export { LineChart } from './LineChart'; diff --git a/datahub-web-react/src/alchemy-components/components/LineChart/types.ts b/datahub-web-react/src/alchemy-components/components/LineChart/types.ts new file mode 100644 index 00000000000000..cf45662ba7cf90 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/LineChart/types.ts @@ -0,0 +1,22 @@ +import { TickFormatter, TickLabelProps } from '@visx/axis'; +import { Margin } from '@visx/xychart'; +import { RenderTooltipGlyphProps } from '@visx/xychart/lib/components/Tooltip'; +import React from 'react'; + +export type LineChartProps = { + data: DatumType[]; + xAccessor: (datum: DatumType) => string | number; + yAccessor: (datum: DatumType) => number; + renderTooltipContent?: (datum: DatumType) => React.ReactNode; + margin?: Margin; + leftAxisTickFormat?: TickFormatter; + leftAxisTickLabelProps?: TickLabelProps; + bottomAxisTickFormat?: TickFormatter; + bottomAxisTickLabelProps?: TickLabelProps; + lineColor?: string; + areaColor?: string; + gridColor?: string; + renderGradients?: () => React.ReactNode; + toolbarVerticalCrosshairStyle?: React.SVGProps; + renderTooltipGlyph?: (props: RenderTooltipGlyphProps) => React.ReactNode | undefined; +}; diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.stories.tsx b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.stories.tsx new file mode 100644 index 00000000000000..7016ecbc7c90a0 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.stories.tsx @@ -0,0 +1,71 @@ +import React from 'react'; + +import type { Meta, StoryObj } from '@storybook/react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import { PageTitle } from '.'; + +// Auto Docs +const meta = { + title: 'Pages / Page Title', + component: PageTitle, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'Used to render the title and subtitle for a page.', + }, + }, + + // Component-level argTypes + argTypes: { + title: { + description: 'The title text', + }, + subTitle: { + description: 'The subtitle text', + }, + variant: { + description: 'The variant of header based on its usage', + }, + }, + + // Define default args + args: { + title: 'Automations', + subTitle: 'Create & manage automations', + variant: 'pageHeader', + }, +} satisfies Meta; + +export default meta; + +// Stories + +type Story = StoryObj; + +// Basic story is what is displayed 1st in storybook +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const withLink = () => ( + + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas aliquet nulla id felis vehicula, et + posuere dui dapibus. Nullam rhoncus massa non tortor convallis, in blandit turpis + rutrum. Morbi tempus velit mauris, at mattis metus mattis sed. Nunc molestie efficitur lectus, vel + mollis eros. + + } + /> +); + +export const sectionHeader = () => ( + +); diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.tsx b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.tsx new file mode 100644 index 00000000000000..3dcf42ff2fc0e2 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/PageTitle/PageTitle.tsx @@ -0,0 +1,17 @@ +import React from 'react'; +import { PageTitleProps } from './types'; +import { Container, SubTitle, Title } from './components'; +import { Pill } from '../Pills'; + +export const PageTitle = ({ title, subTitle, pillLabel, variant = 'pageHeader' }: PageTitleProps) => { + return ( + + + {title} + {pillLabel ? <Pill label={pillLabel} size="sm" clickable={false} /> : null} + + + {subTitle ? {subTitle} : null} + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/components.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/components.ts new file mode 100644 index 00000000000000..328323434e0403 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/PageTitle/components.ts @@ -0,0 +1,52 @@ +import styled from 'styled-components'; +import { typography, colors } from '@components/theme'; +import { getHeaderSubtitleStyles, getHeaderTitleStyles } from './utils'; + +// Text Styles +const titleStyles = { + display: 'flex', + alignItems: 'center', + gap: 8, + fontWeight: typography.fontWeights.bold, + color: colors.gray[600], +}; + +const subTitleStyles = { + fontWeight: typography.fontWeights.normal, + color: colors.gray[1700], +}; + +// Default styles +const baseStyles = { + fontFamily: typography.fonts.body, + margin: 0, + + '& a': { + color: colors.violet[400], + textDecoration: 'none', + transition: 'color 0.15s ease', + + '&:hover': { + color: colors.violet[500], + }, + }, +}; + +export const Container = styled.div` + display: flex; + flex-direction: column; + align-items: start; + justify-content: start; +`; + +export const Title = styled.div<{ variant: string }>(({ variant }) => ({ + ...baseStyles, + ...titleStyles, + ...getHeaderTitleStyles(variant), +})); + +export const SubTitle = styled.div<{ variant: string }>(({ variant }) => ({ + ...baseStyles, + ...subTitleStyles, + ...getHeaderSubtitleStyles(variant), +})); diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/index.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/index.ts new file mode 100644 index 00000000000000..2888306f7c9a66 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/PageTitle/index.ts @@ -0,0 +1 @@ +export { PageTitle } from './PageTitle'; diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/types.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/types.ts new file mode 100644 index 00000000000000..fb1e207d0bbd7b --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/PageTitle/types.ts @@ -0,0 +1,8 @@ +import React from 'react'; + +export interface PageTitleProps { + title: string; + subTitle?: string | React.ReactNode; + pillLabel?: string; + variant?: 'pageHeader' | 'sectionHeader'; +} diff --git a/datahub-web-react/src/alchemy-components/components/PageTitle/utils.ts b/datahub-web-react/src/alchemy-components/components/PageTitle/utils.ts new file mode 100644 index 00000000000000..fe6d18688f31f1 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/PageTitle/utils.ts @@ -0,0 +1,27 @@ +import { typography } from '@components/theme'; + +export const getHeaderTitleStyles = (variant) => { + if (variant === 'sectionHeader') { + return { + fontSize: typography.fontSizes.lg, + lineHeight: typography.lineHeights.lg, + }; + } + return { + fontSize: typography.fontSizes['3xl'], + lineHeight: typography.lineHeights['3xl'], + }; +}; + +export const getHeaderSubtitleStyles = (variant) => { + if (variant === 'sectionHeader') { + return { + fontSize: typography.fontSizes.md, + lineHeight: typography.lineHeights.md, + }; + } + return { + fontSize: typography.fontSizes.lg, + lineHeight: typography.lineHeights.lg, + }; +}; diff --git a/datahub-web-react/src/alchemy-components/components/Pills/Pill.stories.tsx b/datahub-web-react/src/alchemy-components/components/Pills/Pill.stories.tsx new file mode 100644 index 00000000000000..d5cdffef6d6bd3 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Pills/Pill.stories.tsx @@ -0,0 +1,126 @@ +import React from 'react'; + +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import type { Meta, StoryObj } from '@storybook/react'; + +import { GridList } from '@components/.docs/mdx-components'; +import { AVAILABLE_ICONS } from '../Icon'; +import { Pill, pillDefault } from './Pill'; + +const meta = { + title: 'Components / Pill', + component: Pill, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.EXPERIMENTAL], + docs: { + subtitle: 'A component that is used to get pill', + }, + }, + + // Component-level argTypes + argTypes: { + label: { + description: 'Label for the Pill.', + table: { + defaultValue: { summary: pillDefault.label }, + }, + control: { + type: 'text', + }, + }, + leftIcon: { + description: 'The icon to display in the Pill icon.', + type: 'string', + options: AVAILABLE_ICONS, + control: { + type: 'select', + }, + }, + rightIcon: { + description: 'The icon to display in the Pill icon.', + type: 'string', + options: AVAILABLE_ICONS, + control: { + type: 'select', + }, + }, + size: { + description: 'The size of the pill.', + options: ['sm', 'md', 'lg', 'xl'], + table: { + defaultValue: { summary: pillDefault.size }, + }, + control: { + type: 'select', + }, + }, + variant: { + description: 'The size of the Pill.', + options: ['filled', 'outline'], + table: { + defaultValue: { summary: pillDefault.variant }, + }, + control: { + type: 'select', + }, + }, + colorScheme: { + description: 'The color of the Pill.', + options: ['violet', 'green', 'red', 'blue', 'gray'], + table: { + defaultValue: { summary: pillDefault.color }, + }, + control: { + type: 'select', + }, + }, + }, + + // Define defaults + args: { + label: pillDefault.label, + leftIcon: pillDefault.leftIcon, + rightIcon: pillDefault.rightIcon, + size: pillDefault.size, + variant: pillDefault.variant, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const sizes = () => ( + + + + + +); + +export const colors = () => ( + + + + + + + + +); + +export const withIcon = () => ( + + + + + +); diff --git a/datahub-web-react/src/alchemy-components/components/Pills/Pill.tsx b/datahub-web-react/src/alchemy-components/components/Pills/Pill.tsx new file mode 100644 index 00000000000000..898ec89fce5957 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Pills/Pill.tsx @@ -0,0 +1,42 @@ +import { Icon } from '@components'; +import React from 'react'; + +import { PillContainer, PillText } from './components'; +import { PillProps } from './types'; + +export const pillDefault: PillProps = { + label: 'Label', + size: 'md', + variant: 'filled', + clickable: true, +}; + +export function Pill({ + label = pillDefault.label, + size = pillDefault.size, + leftIcon, + rightIcon, + colorScheme, + variant = pillDefault.variant, + clickable = pillDefault.clickable, + id, + onClickRightIcon, + onClickLeftIcon, + onPillClick, +}: PillProps) { + return ( + + {leftIcon && } + {label} + {rightIcon && } + + ); +} diff --git a/datahub-web-react/src/alchemy-components/components/Pills/components.ts b/datahub-web-react/src/alchemy-components/components/Pills/components.ts new file mode 100644 index 00000000000000..79734561a92da6 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Pills/components.ts @@ -0,0 +1,33 @@ +import { spacing } from '@components/theme'; +import styled from 'styled-components'; + +import { PillStyleProps } from './types'; +import { getPillStyle } from './utils'; + +export const PillContainer = styled.div( + // Dynamic styles + (props: PillStyleProps) => ({ ...getPillStyle(props as PillStyleProps) }), + { + // Base root styles + display: 'inline-flex', + alignItems: 'center', + gap: spacing.xxsm, + cursor: 'pointer', + padding: '0px 8px', + borderRadius: '200px', + maxWidth: '100%', + + // Base Disabled styles + '&:disabled': { + cursor: 'not-allowed', + }, + }, +); + +export const PillText = styled.span({ + maxWidth: '100%', + display: 'block', + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', +}); diff --git a/datahub-web-react/src/alchemy-components/components/Pills/index.ts b/datahub-web-react/src/alchemy-components/components/Pills/index.ts new file mode 100644 index 00000000000000..85a76193db2670 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Pills/index.ts @@ -0,0 +1 @@ +export { Pill } from './Pill'; diff --git a/datahub-web-react/src/alchemy-components/components/Pills/types.ts b/datahub-web-react/src/alchemy-components/components/Pills/types.ts new file mode 100644 index 00000000000000..17d4d12465e1ef --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Pills/types.ts @@ -0,0 +1,18 @@ +import { ColorOptions, SizeOptions, VariantOptions } from '@src/alchemy-components/theme/config'; +import { HTMLAttributes } from 'react'; + +export interface PillStyleProps { + colorScheme?: ColorOptions; // need to keep colorScheme because HTMLAttributes also have color property + variant?: VariantOptions; + size?: SizeOptions; + clickable?: boolean; +} + +export interface PillProps extends HTMLAttributes, PillStyleProps { + label: string; + rightIcon?: string; + leftIcon?: string; + onClickRightIcon?: (e: React.MouseEvent) => void; + onClickLeftIcon?: (e: React.MouseEvent) => void; + onPillClick?: (e: React.MouseEvent) => void; +} diff --git a/datahub-web-react/src/alchemy-components/components/Pills/utils.ts b/datahub-web-react/src/alchemy-components/components/Pills/utils.ts new file mode 100644 index 00000000000000..832bf95640982b --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Pills/utils.ts @@ -0,0 +1,147 @@ +import { colors, typography } from '@src/alchemy-components/theme'; +import { getColor, getFontSize } from '@src/alchemy-components/theme/utils'; +import { PillStyleProps } from './types'; + +// Utility function to get color styles for pill - does not generate CSS +const getPillColorStyles = (variant, color) => { + const defaultStyles = { + bgColor: getColor(color, 100), + hoverBgColor: getColor('gray', 100), + borderColor: '', + activeBorderColor: getColor('violet', 500), + textColor: getColor(color, 600), + }; + + const colorOverrides = { + violet: { + textColor: getColor(color, 500), + bgColor: getColor('gray', 1000), + borderColor: 'transparent', + hoverBgColor: getColor(color, 100), + activeBorderColor: getColor(color, 500), + }, + blue: { + textColor: getColor(color, 1000), + bgColor: getColor('gray', 1100), + borderColor: 'transparent', + hoverBgColor: getColor(color, 1100), + activeBorderColor: getColor(color, 1000), + }, + red: { + textColor: getColor(color, 1000), + bgColor: getColor('gray', 1200), + hoverBgColor: getColor(color, 1100), + activeBorderColor: getColor(color, 1000), + }, + green: { + textColor: getColor(color, 1000), + bgColor: getColor('gray', 1300), + hoverBgColor: getColor(color, 1100), + activeBorderColor: getColor(color, 1000), + }, + yellow: { + textColor: getColor(color, 1000), + bgColor: getColor('gray', 1400), + hoverBgColor: getColor(color, 1100), + activeBorderColor: getColor(color, 1000), + }, + }; + + const styles = colorOverrides[color] || defaultStyles; + + if (variant === 'outline') { + return { + bgColor: colors.transparent, + borderColor: getColor('gray', 1400), + textColor: getColor(color, 600), + }; + } + + return styles; +}; + +// Generate variant styles for pill +const getPillVariantStyles = (variant, colorStyles) => + ({ + filled: { + backgroundColor: colorStyles.bgColor, + border: `1px solid transparent`, + color: colorStyles.textColor, + '&:hover': { + backgroundColor: colorStyles.hoverBgColor, + }, + }, + outline: { + backgroundColor: 'transparent', + border: `1px solid ${colorStyles.borderColor}`, + color: colorStyles.textColor, + '&:hover': { + backgroundColor: colorStyles.hoverBgColor, + border: `1px solid transparent`, + }, + '&:disabled': { + border: `1px solid transparent`, + }, + }, + text: { + color: colorStyles.textColor, + }, + }[variant]); + +// Generate font styles for pill +const getPillFontStyles = (size) => { + const baseFontStyles = { + fontFamily: typography.fonts.body, + fontWeight: typography.fontWeights.normal, + lineHeight: typography.lineHeights.none, + }; + + const sizeMap = { + xs: { fontSize: getFontSize(size), lineHeight: '16px' }, + sm: { fontSize: getFontSize(size), lineHeight: '22px' }, + md: { fontSize: getFontSize(size), lineHeight: '24px' }, + lg: { fontSize: getFontSize(size), lineHeight: '30px' }, + xl: { fontSize: getFontSize(size), lineHeight: '34px' }, + }; + + return { + ...baseFontStyles, + ...sizeMap[size], + }; +}; + +// Generate active styles for pill +const getPillActiveStyles = (styleColors) => ({ + borderColor: styleColors.activeBorderColor, +}); + +/* + * Main function to generate styles for pill + */ +export const getPillStyle = (props: PillStyleProps) => { + const { variant, colorScheme = 'gray', size, clickable = true } = props; + + // Get map of colors + const colorStyles = getPillColorStyles(variant, colorScheme); + + // Define styles for pill + let styles = { + ...getPillVariantStyles(variant, colorStyles), + ...getPillFontStyles(size), + '&:focus': { + ...getPillActiveStyles(colorStyles), + outline: 'none', // Remove default browser focus outline if needed + }, + '&:active': { + ...getPillActiveStyles(colorStyles), + }, + }; + if (!clickable) { + styles = { + ...styles, + pointerEvents: 'none', + }; + } + + return styles; +}; diff --git a/datahub-web-react/src/alchemy-components/components/Popover/Popover.tsx b/datahub-web-react/src/alchemy-components/components/Popover/Popover.tsx new file mode 100644 index 00000000000000..8f6ca61976b206 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Popover/Popover.tsx @@ -0,0 +1,6 @@ +import { Popover, PopoverProps } from 'antd'; +import * as React from 'react'; + +export default function DataHubPopover(props: PopoverProps & React.RefAttributes) { + return ; +} diff --git a/datahub-web-react/src/alchemy-components/components/Popover/index.ts b/datahub-web-react/src/alchemy-components/components/Popover/index.ts new file mode 100644 index 00000000000000..02df6c38e8c4ea --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Popover/index.ts @@ -0,0 +1 @@ +export { default as Popover } from './Popover'; diff --git a/datahub-web-react/src/alchemy-components/components/Radio/Radio.stories.tsx b/datahub-web-react/src/alchemy-components/components/Radio/Radio.stories.tsx new file mode 100644 index 00000000000000..cb3116d7b8941b --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Radio/Radio.stories.tsx @@ -0,0 +1,136 @@ +import React from 'react'; +import type { Meta, StoryObj } from '@storybook/react'; +import { GridList } from '@components/.docs/mdx-components'; +import { Radio, radioDefaults, RadioGroup } from './Radio'; +import { Heading } from '../Heading'; +import { RadioProps } from './types'; + +const MOCK_RADIOS: RadioProps[] = [ + { + label: 'Label 1', + error: '', + isChecked: false, + isDisabled: false, + isIntermediate: false, + isRequired: false, + }, + { + label: 'Label 2', + error: '', + isChecked: false, + isDisabled: false, + isIntermediate: false, + isRequired: false, + }, + { + label: 'Label 3', + error: '', + isChecked: false, + isDisabled: false, + isIntermediate: false, + isRequired: false, + }, +]; + +const meta = { + title: 'Forms / Radio', + component: Radio, + parameters: { + layout: 'centered', + docs: { + subtitle: 'A component that is used to get user input in the state of a radio button.', + }, + }, + argTypes: { + label: { + description: 'Label for the Radio.', + table: { + defaultValue: { summary: radioDefaults.label }, + }, + control: { + type: 'text', + }, + }, + error: { + description: 'Enforce error state on the Radio.', + table: { + defaultValue: { summary: radioDefaults.error }, + }, + control: { + type: 'text', + }, + }, + isChecked: { + description: 'Whether the Radio is checked.', + table: { + defaultValue: { summary: radioDefaults?.isChecked?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isDisabled: { + description: 'Whether the Radio is in disabled state.', + table: { + defaultValue: { summary: radioDefaults?.isDisabled?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isRequired: { + description: 'Whether the Radio is a required field.', + table: { + defaultValue: { summary: radioDefaults?.isRequired?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + }, + args: { + label: radioDefaults.label, + error: radioDefaults.error, + isChecked: radioDefaults.isChecked, + isDisabled: radioDefaults.isDisabled, + isRequired: radioDefaults.isRequired, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const states = () => ( + + + + + + +); + +export const disabledStates = () => ( + + + + +); + +export const radioGroups = () => ( + +
+ Horizontal Radio Group + +
+
+ Vertical Radio Group + +
+ +); diff --git a/datahub-web-react/src/alchemy-components/components/Radio/Radio.tsx b/datahub-web-react/src/alchemy-components/components/Radio/Radio.tsx new file mode 100644 index 00000000000000..592c10ec88de8a --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Radio/Radio.tsx @@ -0,0 +1,89 @@ +import React, { useEffect, useState } from 'react'; +import { RadioGroupProps, RadioProps } from './types'; +import { + RadioWrapper, + Checkmark, + HiddenInput, + Label, + Required, + RadioLabel, + RadioBase, + RadioGroupContainer, +} from './components'; + +export const radioDefaults = { + label: 'Label', + error: '', + isChecked: false, + isDisabled: false, + isRequired: false, + isVertical: false, + setIsChecked: () => {}, +}; + +export const Radio = ({ + label = radioDefaults.label, + error = radioDefaults.error, + isChecked = radioDefaults.isChecked, + isDisabled = radioDefaults.isDisabled, + isRequired = radioDefaults.isRequired, + setIsChecked = radioDefaults.setIsChecked, + ...props +}: RadioProps) => { + const [checked, setChecked] = useState(isChecked || false); + + useEffect(() => { + setChecked(isChecked || false); + }, [isChecked]); + + const id = props.id || `checkbox-${label}`; + + return ( + + + { + setChecked(true); + setIsChecked?.(true); + }} + aria-label={label} + aria-labelledby={id} + aria-checked={checked} + {...props} + /> + + + {label && ( + + + + )} + + ); +}; + +export const RadioGroup = ({ isVertical, radios }: RadioGroupProps) => { + if (!radios.length) { + return <>; + } + + return ( + + {radios.map((checkbox) => { + const props = { ...checkbox }; + return ( + + + + ); + })} + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Radio/components.ts b/datahub-web-react/src/alchemy-components/components/Radio/components.ts new file mode 100644 index 00000000000000..027971be179584 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Radio/components.ts @@ -0,0 +1,83 @@ +import { borders, colors, radius, spacing } from '@components/theme'; +import styled from 'styled-components'; +import { formLabelTextStyles } from '../commonStyles'; +import { getRadioBorderColor, getRadioCheckmarkColor } from './utils'; + +export const RadioWrapper = styled.div<{ disabled: boolean; error: string }>(({ disabled, error }) => ({ + position: 'relative', + margin: '20px', + width: '20px', + height: '20px', + border: `${borders['2px']} ${getRadioBorderColor(disabled, error)}`, + backgroundColor: colors.white, + borderRadius: radius.full, + display: 'flex', + justifyContent: 'flex-start', + alignItems: 'center', + marginRight: '40px', + cursor: !disabled ? 'pointer' : 'none', + transition: 'border 0.3s ease, outline 0.3s ease', + '&:hover': { + border: `${borders['2px']} ${!disabled && !error ? colors.violet[500] : getRadioBorderColor(disabled, error)}`, + outline: !disabled && !error ? `${borders['2px']} ${colors.gray[200]}` : 'none', + }, +})); + +export const RadioBase = styled.div({}); + +export const Label = styled.div({ + ...formLabelTextStyles, + display: 'flex', + justifyContent: 'center', + alignItems: 'center', +}); + +export const RadioLabel = styled.div({ + display: 'flex', + justifyContent: 'center', + alignItems: 'center', +}); + +export const Required = styled.span({ + color: colors.red[500], + marginLeft: spacing.xxsm, +}); + +export const RadioHoverState = styled.div({ + border: `${borders['2px']} ${colors.violet[500]}`, + width: 'calc(100% - -3px)', + height: 'calc(100% - -3px)', + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + borderRadius: radius.full, +}); + +export const Checkmark = styled.div<{ checked: boolean; disabled: boolean; error: string }>( + ({ checked, disabled, error }) => ({ + width: 'calc(100% - 6px)', + height: 'calc(100% - 6px)', + borderRadius: radius.full, + background: getRadioCheckmarkColor(checked, disabled, error), + display: checked ? 'inline-block' : 'none', + position: 'absolute', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)', + }), +); + +export const HiddenInput = styled.input<{ checked: boolean }>({ + opacity: 0, + width: '20px', + height: '20px', +}); + +export const RadioGroupContainer = styled.div<{ isVertical?: boolean }>(({ isVertical }) => ({ + display: 'flex', + flexDirection: isVertical ? 'column' : 'row', + justifyContent: 'center', + alignItems: 'center', + gap: !isVertical ? spacing.md : spacing.none, + margin: !isVertical ? spacing.xxsm : spacing.none, +})); diff --git a/datahub-web-react/src/alchemy-components/components/Radio/types.ts b/datahub-web-react/src/alchemy-components/components/Radio/types.ts new file mode 100644 index 00000000000000..59fd15654f916a --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Radio/types.ts @@ -0,0 +1,16 @@ +import { InputHTMLAttributes } from 'react'; + +export interface RadioProps extends InputHTMLAttributes { + label?: string; + error?: string; + isChecked?: boolean; + setIsChecked?: React.Dispatch>; + isDisabled?: boolean; + isIntermediate?: boolean; + isRequired?: boolean; +} + +export interface RadioGroupProps { + isVertical?: boolean; + radios: RadioProps[]; +} diff --git a/datahub-web-react/src/alchemy-components/components/Radio/utils.ts b/datahub-web-react/src/alchemy-components/components/Radio/utils.ts new file mode 100644 index 00000000000000..ed9dcc35d303b4 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Radio/utils.ts @@ -0,0 +1,27 @@ +import { colors } from '@components/theme'; + +const radioBorderColors = { + default: colors.gray[400], + disabled: colors.gray[300], + error: colors.red[500], +}; + +const radioCheckmarkColors = { + default: colors.white, + disabled: colors.gray[300], + checked: colors.violet[500], + error: colors.red[500], +}; + +export function getRadioBorderColor(disabled: boolean, error: string) { + if (disabled) return radioBorderColors.disabled; + if (error) return radioCheckmarkColors.error; + return radioBorderColors.default; +} + +export function getRadioCheckmarkColor(checked: boolean, disabled: boolean, error: string) { + if (disabled) return radioCheckmarkColors.disabled; + if (error) return radioCheckmarkColors.error; + if (checked) return radioCheckmarkColors.checked; + return radioCheckmarkColors.default; +} diff --git a/datahub-web-react/src/alchemy-components/components/Select/BasicSelect.tsx b/datahub-web-react/src/alchemy-components/components/Select/BasicSelect.tsx new file mode 100644 index 00000000000000..b49159ba38a758 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/BasicSelect.tsx @@ -0,0 +1,339 @@ +import { Button, Icon, Pill, Text } from '@components'; +import { isEqual } from 'lodash'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + ActionButtonsContainer, + Container, + DescriptionContainer, + Dropdown, + FooterBase, + LabelContainer, + LabelsWrapper, + OptionContainer, + OptionLabel, + OptionList, + Placeholder, + SearchIcon, + SearchInput, + SearchInputContainer, + SelectAllOption, + SelectBase, + SelectLabel, + SelectValue, + StyledCancelButton, + StyledCheckbox, + StyledClearButton, +} from './components'; +import { ActionButtonsProps, SelectLabelDisplayProps, SelectOption, SelectProps } from './types'; +import { getFooterButtonSize } from './utils'; + +const SelectLabelDisplay = ({ + selectedValues, + options, + placeholder, + isMultiSelect, + removeOption, + disabledValues, + showDescriptions, +}: SelectLabelDisplayProps) => { + const selectedOptions = options.filter((opt) => selectedValues.includes(opt.value)); + return ( + + {!!selectedOptions.length && + isMultiSelect && + selectedOptions.map((o) => { + const isDisabled = disabledValues?.includes(o.value); + return ( + { + e.stopPropagation(); + removeOption?.(o); + }} + clickable={!isDisabled} + /> + ); + })} + {!selectedValues.length && {placeholder}} + {!isMultiSelect && ( + <> + {selectedOptions[0]?.label} + {showDescriptions && !!selectedValues.length && ( + {selectedOptions[0]?.description} + )} + + )} + + ); +}; + +const SelectActionButtons = ({ + selectedValues, + isOpen, + isDisabled, + isReadOnly, + showClear, + handleClearSelection, + fontSize = 'md', +}: ActionButtonsProps) => { + return ( + + {showClear && selectedValues.length > 0 && !isDisabled && !isReadOnly && ( + + )} + + + ); +}; + +// Updated main component +export const selectDefaults: SelectProps = { + options: [], + label: '', + size: 'md', + showSearch: false, + isDisabled: false, + isReadOnly: false, + isRequired: false, + isMultiSelect: false, + showClear: false, + placeholder: 'Select an option', + showSelectAll: false, + selectAllLabel: 'Select All', + showDescriptions: false, +}; + +export const BasicSelect = ({ + options = selectDefaults.options, + label = selectDefaults.label, + values = [], + onCancel, + onUpdate, + showSearch = selectDefaults.showSearch, + isDisabled = selectDefaults.isDisabled, + isReadOnly = selectDefaults.isReadOnly, + isRequired = selectDefaults.isRequired, + showClear = selectDefaults.showClear, + size = selectDefaults.size, + isMultiSelect = selectDefaults.isMultiSelect, + placeholder = selectDefaults.placeholder, + disabledValues = [], + showSelectAll = selectDefaults.showSelectAll, + selectAllLabel = selectDefaults.selectAllLabel, + showDescriptions = selectDefaults.showDescriptions, + ...props +}: SelectProps) => { + const [searchQuery, setSearchQuery] = useState(''); + const [isOpen, setIsOpen] = useState(false); + const [selectedValues, setSelectedValues] = useState(values); + const [tempValues, setTempValues] = useState(values); + const selectRef = useRef(null); + const [areAllSelected, setAreAllSelected] = useState(false); + + useEffect(() => { + if (values?.length > 0 && !isEqual(selectedValues, values)) { + setSelectedValues(values); + } + }, [values, selectedValues]); + + useEffect(() => { + setAreAllSelected(tempValues.length === options.length); + }, [options, tempValues]); + + const filteredOptions = useMemo( + () => options.filter((option) => option.label.toLowerCase().includes(searchQuery.toLowerCase())), + [options, searchQuery], + ); + + const handleDocumentClick = useCallback((e: MouseEvent) => { + if (selectRef.current && !selectRef.current.contains(e.target as Node)) { + setIsOpen(false); + } + }, []); + + useEffect(() => { + document.addEventListener('click', handleDocumentClick); + return () => { + document.removeEventListener('click', handleDocumentClick); + }; + }, [handleDocumentClick]); + + const handleSelectClick = useCallback(() => { + if (!isDisabled && !isReadOnly) { + setTempValues(selectedValues); + setIsOpen((prev) => !prev); + } + }, [isDisabled, isReadOnly, selectedValues]); + + const handleOptionChange = useCallback( + (option: SelectOption) => { + const updatedValues = tempValues.includes(option.value) + ? tempValues.filter((val) => val !== option.value) + : [...tempValues, option.value]; + + setTempValues(isMultiSelect ? updatedValues : [option.value]); + }, + [tempValues, isMultiSelect], + ); + + const removeOption = useCallback( + (option: SelectOption) => { + const updatedValues = selectedValues.filter((val) => val !== option.value); + setSelectedValues(updatedValues); + }, + [selectedValues], + ); + + const handleUpdateClick = useCallback(() => { + setSelectedValues(tempValues); + setIsOpen(false); + if (onUpdate) { + onUpdate(tempValues); + } + }, [tempValues, onUpdate]); + + const handleCancelClick = useCallback(() => { + setIsOpen(false); + setTempValues(selectedValues); + if (onCancel) { + onCancel(); + } + }, [selectedValues, onCancel]); + + const handleClearSelection = useCallback(() => { + setSelectedValues([]); + setAreAllSelected(false); + setTempValues([]); + setIsOpen(false); + if (onUpdate) { + onUpdate([]); + } + }, [onUpdate]); + + const handleSelectAll = () => { + if (areAllSelected) { + setTempValues([]); + onUpdate?.([]); + } else { + const allValues = options.map((option) => option.value); + setTempValues(allValues); + onUpdate?.(allValues); + } + setAreAllSelected(!areAllSelected); + }; + + return ( + + {label && {label}} + + + + + {isOpen && ( + + {showSearch && ( + + setSearchQuery(e.target.value)} + style={{ fontSize: size || 'md' }} + /> + + + )} + + {showSelectAll && isMultiSelect && ( + !(disabledValues.length === options.length) && handleSelectAll()} + isDisabled={disabledValues.length === options.length} + > + + {selectAllLabel} + + + + )} + {filteredOptions.map((option) => ( + !isMultiSelect && handleOptionChange(option)} + isSelected={tempValues.includes(option.value)} + isMultiSelect={isMultiSelect} + isDisabled={disabledValues?.includes(option.value)} + > + {isMultiSelect ? ( + + {option.label} + handleOptionChange(option)} + checked={tempValues.includes(option.value)} + disabled={disabledValues?.includes(option.value)} + /> + + ) : ( + + + {option.label} + + {!!option.description && ( + + {option.description} + + )} + + )} + + ))} + + + + Cancel + + + + + )} + + ); +}; + +export default BasicSelect; diff --git a/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedOption.tsx b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedOption.tsx new file mode 100644 index 00000000000000..8a7d3670b2b1b9 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedOption.tsx @@ -0,0 +1,309 @@ +import React, { useState, useMemo, useEffect } from 'react'; + +import { colors, Icon } from '@components'; +import theme from '@components/theme'; +import styled from 'styled-components'; +import { Checkbox } from 'antd'; + +import { OptionLabel } from '../components'; +import { SelectOption } from './types'; + +const ParentOption = styled.div` + display: flex; + align-items: center; +`; + +const ChildOptions = styled.div` + padding-left: 20px; +`; + +const StyledCheckbox = styled(Checkbox)<{ checked: boolean; indeterminate?: boolean }>` + .ant-checkbox-inner { + border: 1px solid ${colors.gray[300]} !important; + border-radius: 3px; + } + margin-left: auto; + ${(props) => + props.checked && + !props.indeterminate && + ` + .ant-checkbox-inner { + background-color: ${theme.semanticTokens.colors.primary}; + border-color: ${theme.semanticTokens.colors.primary} !important; + } + `} + ${(props) => + props.indeterminate && + ` + .ant-checkbox-inner { + &:after { + background-color: ${theme.semanticTokens.colors.primary}; + } + } + `} + ${(props) => + props.disabled && + ` + .ant-checkbox-inner { + background-color: ${colors.gray[200]} !important; + } + `} +`; + +function getChildrenRecursively( + directChildren: SelectOption[], + parentValueToOptions: { [parentValue: string]: SelectOption[] }, +) { + const visitedParents = new Set(); + let allChildren: SelectOption[] = []; + + function getChildren(parentValue: string) { + const newChildren = parentValueToOptions[parentValue] || []; + if (visitedParents.has(parentValue) || !newChildren.length) { + return; + } + + visitedParents.add(parentValue); + allChildren = [...allChildren, ...newChildren]; + newChildren.forEach((child) => getChildren(child.value || child.value)); + } + + directChildren.forEach((c) => getChildren(c.value || c.value)); + + return allChildren; +} + +interface OptionProps { + option: SelectOption; + selectedOptions: SelectOption[]; + parentValueToOptions: { [parentValue: string]: SelectOption[] }; + areParentsSelectable: boolean; + handleOptionChange: (node: SelectOption) => void; + addOptions: (nodes: SelectOption[]) => void; + removeOptions: (nodes: SelectOption[]) => void; + loadData?: (node: SelectOption) => void; + isMultiSelect?: boolean; + isLoadingParentChildList?: boolean; + setSelectedOptions: React.Dispatch>; +} + +export const NestedOption = ({ + option, + selectedOptions, + parentValueToOptions, + handleOptionChange, + addOptions, + removeOptions, + loadData, + isMultiSelect, + areParentsSelectable, + isLoadingParentChildList, + setSelectedOptions, +}: OptionProps) => { + const [autoSelectChildren, setAutoSelectChildren] = useState(false); + const [loadingParentUrns, setLoadingParentUrns] = useState([]); + const [isOpen, setIsOpen] = useState(false); + const directChildren = useMemo( + () => parentValueToOptions[option.value] || [], + [parentValueToOptions, option.value], + ); + + const recursiveChildren = useMemo( + () => getChildrenRecursively(directChildren, parentValueToOptions), + [directChildren, parentValueToOptions], + ); + + const children = useMemo(() => [...directChildren, ...recursiveChildren], [directChildren, recursiveChildren]); + const selectableChildren = useMemo( + () => (areParentsSelectable ? children : children.filter((c) => !c.isParent)), + [areParentsSelectable, children], + ); + const parentChildren = useMemo(() => children.filter((c) => c.isParent), [children]); + + useEffect(() => { + if (autoSelectChildren && selectableChildren.length) { + addOptions(selectableChildren); + setAutoSelectChildren(false); + } + }, [autoSelectChildren, selectableChildren, addOptions]); + + const areAllChildrenSelected = useMemo( + () => selectableChildren.every((child) => selectedOptions.find((o) => o.value === child.value)), + [selectableChildren, selectedOptions], + ); + + const areAnyChildrenSelected = useMemo( + () => selectableChildren.some((child) => selectedOptions.find((o) => o.value === child.value)), + [selectableChildren, selectedOptions], + ); + + const areAnyUnselectableChildrenUnexpanded = !!parentChildren.find( + (parent) => !selectableChildren.find((child) => child.parentValue === parent.value), + ); + + const isSelected = useMemo( + () => + !!selectedOptions.find((o) => o.value === option.value) || + (!areParentsSelectable && + !!option.isParent && + !!selectableChildren.length && + areAllChildrenSelected && + !areAnyUnselectableChildrenUnexpanded), + [ + selectedOptions, + areAllChildrenSelected, + areAnyUnselectableChildrenUnexpanded, + areParentsSelectable, + option.isParent, + option.value, + selectableChildren.length, + ], + ); + + const isImplicitlySelected = useMemo( + () => !option.isParent && !!selectedOptions.find((o) => o.value === option.parentValue), + [selectedOptions, option.isParent, option.parentValue], + ); + + const isParentMissingChildren = useMemo(() => !!option.isParent && !children.length, [children, option.isParent]); + + const isPartialSelected = useMemo( + () => + (!areAllChildrenSelected && areAnyChildrenSelected) || + (isSelected && isParentMissingChildren) || + (isSelected && areAnyUnselectableChildrenUnexpanded) || + (areAnyUnselectableChildrenUnexpanded && areAnyChildrenSelected) || + (isSelected && !!children.length && !areAnyChildrenSelected), + [ + isSelected, + children, + areAllChildrenSelected, + areAnyChildrenSelected, + areAnyUnselectableChildrenUnexpanded, + isParentMissingChildren, + ], + ); + + const selectOption = () => { + if (areParentsSelectable && option.isParent) { + const existingSelectedOptions = new Set(selectedOptions.map((opt) => opt.value)); + const existingChildSelectedOptions = + selectedOptions.filter((opt) => opt.parentValue === option.value) || []; + if (existingSelectedOptions.has(option.value)) { + removeOptions([option]); + } else { + // filter out the childrens of parent selection as we are allowing implicitly selection + const filteredOptions = selectedOptions.filter( + (selectedOption) => !existingChildSelectedOptions.find((o) => o.value === selectedOption.value), + ); + const newSelectedOptions = [...filteredOptions, option]; + + setSelectedOptions(newSelectedOptions); + } + } else if (isPartialSelected || (!isSelected && !areAnyChildrenSelected)) { + const optionsToAdd = option.isParent && !areParentsSelectable ? selectableChildren : [option]; + addOptions(optionsToAdd); + } else if (areAllChildrenSelected) { + removeOptions([option, ...selectableChildren]); + } else { + handleOptionChange(option); + } + }; + + // one loader variable for fetching data for expanded parents and their respective child nodes + useEffect(() => { + // once loading has been done just remove all the parent node urn + if (!isLoadingParentChildList) { + setLoadingParentUrns([]); + } + }, [isLoadingParentChildList]); + + return ( +
+ + { + e.preventDefault(); + if (isImplicitlySelected) { + return; + } + if (isParentMissingChildren) { + setLoadingParentUrns((previousIds) => [...previousIds, option.value]); + loadData?.(option); + } + if (option.isParent) { + setIsOpen(!isOpen); + } else { + selectOption(); + } + }} + isSelected={!isMultiSelect && isSelected} + // added hack to show cursor in wait untill we get the inline spinner + style={{ width: '100%', cursor: loadingParentUrns.includes(option.value) ? 'wait' : 'pointer' }} + > + {option.isParent && {option.label}} + {!option.isParent && <>{option.label}} + {option.isParent && ( + { + e.stopPropagation(); + e.preventDefault(); + setIsOpen(!isOpen); + if (!isOpen && isParentMissingChildren) { + setLoadingParentUrns((previousIds) => [...previousIds, option.value]); + loadData?.(option); + } + }} + icon="ChevronLeft" + rotate={isOpen ? '90' : '270'} + size="xl" + color="gray" + style={{ cursor: 'pointer', marginLeft: '4px' }} + /> + )} + { + e.preventDefault(); + if (isImplicitlySelected) { + return; + } + e.stopPropagation(); + if (isParentMissingChildren) { + loadData?.(option); + if (!areParentsSelectable) { + setAutoSelectChildren(true); + } + } + selectOption(); + }} + disabled={isImplicitlySelected} + /> + + + {isOpen && ( + + {directChildren.map((child) => ( + + ))} + + )} +
+ ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedSelect.tsx b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedSelect.tsx new file mode 100644 index 00000000000000..744c7bfcfec0d2 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/Nested/NestedSelect.tsx @@ -0,0 +1,312 @@ +import React, { useCallback, useEffect, useRef, useState } from 'react'; +import styled from 'styled-components'; + +import { Icon, Pill } from '@components'; + +import { + ActionButtonsContainer, + Container, + Dropdown, + OptionList, + Placeholder, + SearchIcon, + SearchInput, + SearchInputContainer, + SelectBase, + SelectLabel, + StyledClearButton, +} from '../components'; + +import { SelectSizeOptions } from '../types'; +import { NestedOption } from './NestedOption'; +import { SelectOption } from './types'; + +const NO_PARENT_VALUE = 'no_parent_value'; + +const LabelDisplayWrapper = styled.div` + display: flex; + flex-wrap: wrap; + gap: 4px; + max-height: 125px; + min-height: 16px; +`; + +interface SelectLabelDisplayProps { + selectedOptions: SelectOption[]; + placeholder: string; + handleOptionChange: (node: SelectOption) => void; +} + +const SelectLabelDisplay = ({ selectedOptions, placeholder, handleOptionChange }: SelectLabelDisplayProps) => { + return ( + + {!!selectedOptions.length && + selectedOptions.map((o) => ( + { + e.stopPropagation(); + handleOptionChange(o); + }} + /> + ))} + {!selectedOptions.length && {placeholder}} + + ); +}; + +export interface ActionButtonsProps { + fontSize?: SelectSizeOptions; + selectedOptions: SelectOption[]; + isOpen: boolean; + isDisabled: boolean; + isReadOnly: boolean; + handleClearSelection: () => void; +} + +const SelectActionButtons = ({ + selectedOptions, + isOpen, + isDisabled, + isReadOnly, + handleClearSelection, + fontSize = 'md', +}: ActionButtonsProps) => { + return ( + + {!!selectedOptions.length && !isDisabled && !isReadOnly && ( + + )} + + + ); +}; + +export interface SelectProps { + options: SelectOption[]; + label: string; + value?: string; + initialValues?: SelectOption[]; + onCancel?: () => void; + onUpdate?: (selectedValues: SelectOption[]) => void; + size?: SelectSizeOptions; + showSearch?: boolean; + isDisabled?: boolean; + isReadOnly?: boolean; + isRequired?: boolean; + isMultiSelect?: boolean; + areParentsSelectable?: boolean; + loadData?: (node: SelectOption) => void; + onSearch?: (query: string) => void; + width?: number | 'full'; + height?: number; + placeholder?: string; + searchPlaceholder?: string; + isLoadingParentChildList?: boolean; +} + +export const selectDefaults: SelectProps = { + options: [], + label: '', + size: 'md', + showSearch: false, + isDisabled: false, + isReadOnly: false, + isRequired: false, + isMultiSelect: false, + width: 255, + height: 425, +}; + +export const NestedSelect = ({ + options = selectDefaults.options, + label = selectDefaults.label, + initialValues = [], + onUpdate, + loadData, + onSearch, + showSearch = selectDefaults.showSearch, + isDisabled = selectDefaults.isDisabled, + isReadOnly = selectDefaults.isReadOnly, + isRequired = selectDefaults.isRequired, + isMultiSelect = selectDefaults.isMultiSelect, + size = selectDefaults.size, + areParentsSelectable = true, + placeholder, + searchPlaceholder, + height = selectDefaults.height, + isLoadingParentChildList = false, + ...props +}: SelectProps) => { + const [searchQuery, setSearchQuery] = useState(''); + const [isOpen, setIsOpen] = useState(false); + const [selectedOptions, setSelectedOptions] = useState(initialValues); + const selectRef = useRef(null); + + // TODO: handle searching inside of a nested component on the FE only + + const handleDocumentClick = useCallback((e: MouseEvent) => { + if (selectRef.current && !selectRef.current.contains(e.target as Node)) { + setIsOpen(false); + } + }, []); + + useEffect(() => { + document.addEventListener('click', handleDocumentClick); + return () => { + document.removeEventListener('click', handleDocumentClick); + }; + }, [handleDocumentClick]); + + const handleSelectClick = useCallback(() => { + if (!isDisabled && !isReadOnly) { + setIsOpen((prev) => !prev); + } + }, [isDisabled, isReadOnly]); + + const handleSearch = useCallback( + (query: string) => { + setSearchQuery(query); + onSearch?.(query); + }, + [onSearch], + ); + + // Instead of calling the update function individually whenever selectedOptions changes, + // we use the useEffect hook to trigger the onUpdate function automatically when selectedOptions is updated. + useEffect(() => { + if (onUpdate) { + onUpdate(selectedOptions); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedOptions]); + + const handleOptionChange = useCallback( + (option: SelectOption) => { + let newSelectedOptions: SelectOption[]; + if (selectedOptions.find((o) => o.value === option.value)) { + newSelectedOptions = selectedOptions.filter((o) => o.value !== option.value); + } else { + newSelectedOptions = [...selectedOptions, option]; + } + setSelectedOptions(newSelectedOptions); + if (!isMultiSelect) { + setIsOpen(false); + } + }, + [selectedOptions, isMultiSelect], + ); + + const addOptions = useCallback( + (optionsToAdd: SelectOption[]) => { + const existingValues = new Set(selectedOptions.map((option) => option.value)); + const filteredOptionsToAdd = optionsToAdd.filter((option) => !existingValues.has(option.value)); + if (filteredOptionsToAdd.length) { + const newSelectedOptions = [...selectedOptions, ...filteredOptionsToAdd]; + setSelectedOptions(newSelectedOptions); + } + }, + [selectedOptions], + ); + + const removeOptions = useCallback( + (optionsToRemove: SelectOption[]) => { + const newValues = selectedOptions.filter( + (selectedOption) => !optionsToRemove.find((o) => o.value === selectedOption.value), + ); + setSelectedOptions(newValues); + }, + [selectedOptions], + ); + + const handleClearSelection = useCallback(() => { + setSelectedOptions([]); + setIsOpen(false); + if (onUpdate) { + onUpdate([]); + } + }, [onUpdate]); + + // generate map for options to quickly fetch children + const parentValueToOptions: { [parentValue: string]: SelectOption[] } = {}; + options.forEach((o) => { + const parentValue = o.parentValue || NO_PARENT_VALUE; + parentValueToOptions[parentValue] = parentValueToOptions[parentValue] + ? [...parentValueToOptions[parentValue], o] + : [o]; + }); + + const rootOptions = parentValueToOptions[NO_PARENT_VALUE] || []; + + return ( + + {label && {label}} + + + + + {isOpen && ( + + {showSearch && ( + + handleSearch(e.target.value)} + style={{ fontSize: size || 'md', width: '100%' }} + /> + + + )} + + {rootOptions.map((option) => ( + + ))} + + + )} + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Select/Nested/types.ts b/datahub-web-react/src/alchemy-components/components/Select/Nested/types.ts new file mode 100644 index 00000000000000..62d4541fce0d3d --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/Nested/types.ts @@ -0,0 +1,9 @@ +import { Entity } from '@src/types.generated'; + +export interface SelectOption { + value: string; + label: string; + parentValue?: string; + isParent?: boolean; + entity?: Entity; +} diff --git a/datahub-web-react/src/alchemy-components/components/Select/Select.stories.tsx b/datahub-web-react/src/alchemy-components/components/Select/Select.stories.tsx new file mode 100644 index 00000000000000..0ec20b15e771ab --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/Select.stories.tsx @@ -0,0 +1,431 @@ +import { GridList } from '@components/.docs/mdx-components'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import type { Meta, StoryObj } from '@storybook/react'; +import React from 'react'; +import { Select, selectDefaults } from './Select'; +import { SimpleSelect } from './SimpleSelect'; +import { SelectSizeOptions } from './types'; + +// Auto Docs +const meta: Meta = { + title: 'Forms / Select', + component: Select, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'This component allows users to select one or multiple input options from a dropdown list.', + }, + }, + + // Component-level argTypes + argTypes: { + options: { + description: 'Array of options for the Select component.', + control: { + type: 'object', + }, + table: { + defaultValue: { summary: JSON.stringify(selectDefaults.options) }, + }, + }, + label: { + description: 'Label for the Select component.', + control: { + type: 'text', + }, + table: { + defaultValue: { summary: selectDefaults.label }, + }, + }, + values: { + description: 'Selected values for the Select component.', + control: { + type: 'object', + }, + table: { + defaultValue: { summary: selectDefaults.values?.toString() }, + }, + }, + showSearch: { + description: 'Whether to show the search input.', + control: { + type: 'boolean', + }, + table: { + defaultValue: { summary: selectDefaults.showSearch?.toString() }, + }, + }, + isDisabled: { + description: 'Whether the Select component is disabled.', + control: { + type: 'boolean', + }, + table: { + defaultValue: { summary: selectDefaults.isDisabled?.toString() }, + }, + }, + isReadOnly: { + description: 'Whether the Select component is read-only.', + control: { + type: 'boolean', + }, + table: { + defaultValue: { summary: selectDefaults.isReadOnly?.toString() }, + }, + }, + isRequired: { + description: 'Whether the Select component is required.', + control: { + type: 'boolean', + }, + table: { + defaultValue: { summary: selectDefaults.isRequired?.toString() }, + }, + }, + size: { + description: 'Size of the Select component.', + control: { + type: 'select', + options: ['sm', 'md', 'lg'], + }, + table: { + defaultValue: { summary: selectDefaults.size }, + }, + }, + width: { + description: 'Width of the Select component.', + control: { + type: 'number', + }, + table: { + defaultValue: { summary: `${selectDefaults.width}` }, + }, + }, + isMultiSelect: { + description: 'Whether the Select component allows multiple values to be selected.', + control: { + type: 'boolean', + }, + table: { + defaultValue: { summary: selectDefaults.isMultiSelect?.toString() }, + }, + }, + placeholder: { + description: 'Placeholder for the Select component.', + control: { + type: 'text', + }, + table: { + defaultValue: { summary: selectDefaults.placeholder }, + }, + }, + disabledValues: { + description: 'Disabled values for the multi-select component.', + control: { + type: 'object', + }, + table: { + defaultValue: { summary: selectDefaults.disabledValues?.toString() }, + }, + }, + showSelectAll: { + description: 'Whether the multi select component shows Select All button.', + control: { + type: 'boolean', + }, + table: { + defaultValue: { summary: selectDefaults.showSelectAll?.toString() }, + }, + }, + selectAllLabel: { + description: 'Label for Select All button.', + control: { + type: 'text', + }, + table: { + defaultValue: { summary: selectDefaults.selectAllLabel }, + }, + }, + showDescriptions: { + description: 'Whether to show descriptions with the select options.', + control: { + type: 'boolean', + }, + table: { + defaultValue: { summary: selectDefaults.showDescriptions?.toString() }, + }, + }, + }, + + // Define defaults + args: { + options: [ + { label: 'Option 1', value: '1' }, + { label: 'Option 2', value: '2' }, + { label: 'Option 3', value: '3' }, + ], + label: 'Select Label', + values: undefined, + showSearch: selectDefaults.showSearch, + isDisabled: selectDefaults.isDisabled, + isReadOnly: selectDefaults.isReadOnly, + isRequired: selectDefaults.isRequired, + onCancel: () => console.log('Cancel clicked'), + onUpdate: (selectedValues: string[]) => console.log('Update clicked', selectedValues), + size: 'md', // Default size + width: 255, + isMultiSelect: selectDefaults.isMultiSelect, + placeholder: selectDefaults.placeholder, + disabledValues: undefined, + showSelectAll: false, + selectAllLabel: 'Select All', + showDescriptions: false, + }, +} satisfies Meta; + +export default meta; + +// Stories + +type Story = StoryObj; + +const sizeOptions: SelectSizeOptions[] = ['sm', 'md', 'lg']; + +export const simpleSelectSandbox: Story = { + tags: ['dev'], + + render: (props) => ( + + ), +}; + +export const simpleSelectStates = () => ( + + <> + + + + + +); + +export const simpleSelectWithSearch = () => ( + +); + +export const simpleSelectWithMultiSelect = () => ( + +); + +export const simpleSelectWithDisabledValues = () => ( + +); + +export const simpleSelectWithSelectAll = () => ( + +); + +export const simpleSelectWithDescriptions = () => ( + +); + +export const simpleSelectSizes = () => ( + + {sizeOptions.map((size, index) => ( + + ))} + +); + +// Basic story is what is displayed 1st in storybook & is used as the code sandbox +// Pass props to this so that it can be customized via the UI props panel +export const BasicSelectSandbox: Story = { + tags: ['dev'], + + render: (props) => ( + + + + +); + +export const withSearch = () => ( + +); + +export const sizes = () => ( + + {sizeOptions.map((size, index) => ( + alert('Cancel clicked')} + onUpdate={(selectedValues) => alert(`Update clicked with values: ${selectedValues}`)} + size="md" + /> + +); diff --git a/datahub-web-react/src/alchemy-components/components/Select/Select.tsx b/datahub-web-react/src/alchemy-components/components/Select/Select.tsx new file mode 100644 index 00000000000000..da28f090565431 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/Select.tsx @@ -0,0 +1,65 @@ +import React from 'react'; +import { BasicSelect } from './BasicSelect'; +import { SelectProps } from './types'; + +export const selectDefaults: SelectProps = { + options: [], + label: '', + showSearch: false, + values: undefined, + size: 'md', + isDisabled: false, + isReadOnly: false, + isRequired: false, + width: 255, + isMultiSelect: false, + placeholder: 'Select an option', + disabledValues: undefined, + showSelectAll: false, + selectAllLabel: 'Select All', + showDescriptions: false, +}; + +export const Select = ({ + options = selectDefaults.options, + label = selectDefaults.label, + values = [], + onCancel, + onUpdate, + showSearch = selectDefaults.showSearch, + isDisabled = selectDefaults.isDisabled, + isReadOnly = selectDefaults.isReadOnly, + isRequired = selectDefaults.isRequired, + size = selectDefaults.size, + width = selectDefaults.width, + isMultiSelect = selectDefaults.isMultiSelect, + placeholder = selectDefaults.placeholder, + disabledValues = selectDefaults.disabledValues, + showSelectAll = selectDefaults.showSelectAll, + selectAllLabel = selectDefaults.selectAllLabel, + showDescriptions = selectDefaults.showDescriptions, + ...props +}: SelectProps) => { + return ( + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Select/SimpleSelect.tsx b/datahub-web-react/src/alchemy-components/components/Select/SimpleSelect.tsx new file mode 100644 index 00000000000000..be1184cee9e9f5 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/SimpleSelect.tsx @@ -0,0 +1,299 @@ +import { Icon, Pill, Text } from '@components'; +import { isEqual } from 'lodash'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { + ActionButtonsContainer, + Container, + DescriptionContainer, + Dropdown, + LabelContainer, + LabelsWrapper, + OptionContainer, + OptionLabel, + OptionList, + Placeholder, + SearchIcon, + SearchInput, + SearchInputContainer, + SelectAllOption, + SelectBase, + SelectLabel, + SelectValue, + StyledCheckbox, + StyledClearButton, +} from './components'; +import { ActionButtonsProps, SelectLabelDisplayProps, SelectOption, SelectProps } from './types'; + +const SelectLabelDisplay = ({ + selectedValues, + options, + placeholder, + isMultiSelect, + removeOption, + disabledValues, + showDescriptions, +}: SelectLabelDisplayProps) => { + const selectedOptions = options.filter((opt) => selectedValues.includes(opt.value)); + return ( + + {!!selectedOptions.length && + isMultiSelect && + selectedOptions.map((o) => { + const isDisabled = disabledValues?.includes(o.value); + return ( + { + e.stopPropagation(); + removeOption?.(o); + }} + clickable={!isDisabled} + /> + ); + })} + {!selectedValues.length && {placeholder}} + {!isMultiSelect && ( + <> + {selectedOptions[0]?.label} + {showDescriptions && !!selectedValues.length && ( + {selectedOptions[0]?.description} + )} + + )} + + ); +}; + +const SelectActionButtons = ({ + selectedValues, + isOpen, + isDisabled, + isReadOnly, + showClear, + handleClearSelection, + fontSize = 'md', +}: ActionButtonsProps) => { + return ( + + {showClear && selectedValues.length > 0 && !isDisabled && !isReadOnly && ( + + )} + + + ); +}; + +export const selectDefaults: SelectProps = { + options: [], + label: '', + size: 'md', + showSearch: false, + isDisabled: false, + isReadOnly: false, + isRequired: false, + showClear: true, + width: 255, + isMultiSelect: false, + placeholder: 'Select an option ', + showSelectAll: false, + selectAllLabel: 'Select All', + showDescriptions: false, +}; + +export const SimpleSelect = ({ + options = selectDefaults.options, + label = selectDefaults.label, + values = [], + onUpdate, + showSearch = selectDefaults.showSearch, + isDisabled = selectDefaults.isDisabled, + isReadOnly = selectDefaults.isReadOnly, + isRequired = selectDefaults.isRequired, + showClear = selectDefaults.showClear, + size = selectDefaults.size, + isMultiSelect = selectDefaults.isMultiSelect, + placeholder = selectDefaults.placeholder, + disabledValues = [], + showSelectAll = selectDefaults.showSelectAll, + selectAllLabel = selectDefaults.selectAllLabel, + optionListTestId, + showDescriptions = selectDefaults.showDescriptions, + ...props +}: SelectProps) => { + const [searchQuery, setSearchQuery] = useState(''); + const [isOpen, setIsOpen] = useState(false); + const [selectedValues, setSelectedValues] = useState(values); + const selectRef = useRef(null); + const [areAllSelected, setAreAllSelected] = useState(false); + + useEffect(() => { + if (values?.length > 0 && !isEqual(selectedValues, values)) { + setSelectedValues(values); + } + }, [values, selectedValues]); + + useEffect(() => { + setAreAllSelected(selectedValues.length === options.length); + }, [options, selectedValues]); + + const filteredOptions = useMemo( + () => options.filter((option) => option.label.toLowerCase().includes(searchQuery.toLowerCase())), + [options, searchQuery], + ); + + const handleDocumentClick = useCallback((e: MouseEvent) => { + if (selectRef.current && !selectRef.current.contains(e.target as Node)) { + setIsOpen(false); + } + }, []); + + useEffect(() => { + document.addEventListener('click', handleDocumentClick); + return () => { + document.removeEventListener('click', handleDocumentClick); + }; + }, [handleDocumentClick]); + + const handleSelectClick = useCallback(() => { + if (!isDisabled && !isReadOnly) { + setIsOpen((prev) => !prev); + } + }, [isDisabled, isReadOnly]); + + const handleOptionChange = useCallback( + (option: SelectOption) => { + const updatedValues = selectedValues.includes(option.value) + ? selectedValues.filter((val) => val !== option.value) + : [...selectedValues, option.value]; + + setSelectedValues(isMultiSelect ? updatedValues : [option.value]); + if (onUpdate) { + onUpdate(isMultiSelect ? updatedValues : [option.value]); + } + if (!isMultiSelect) setIsOpen(false); + }, + [onUpdate, isMultiSelect, selectedValues], + ); + + const handleClearSelection = useCallback(() => { + setSelectedValues([]); + setAreAllSelected(false); + setIsOpen(false); + if (onUpdate) { + onUpdate([]); + } + }, [onUpdate]); + + const handleSelectAll = () => { + if (areAllSelected) { + setSelectedValues([]); + onUpdate?.([]); + } else { + const allValues = options.map((option) => option.value); + setSelectedValues(allValues); + onUpdate?.(allValues); + } + setAreAllSelected(!areAllSelected); + }; + + return ( + + {label && {label}} + + + + + {isOpen && ( + + {showSearch && ( + + setSearchQuery(e.target.value)} + style={{ fontSize: size || 'md' }} + /> + + + )} + + {showSelectAll && isMultiSelect && ( + !(disabledValues.length === options.length) && handleSelectAll()} + isDisabled={disabledValues.length === options.length} + > + + {selectAllLabel} + + + + )} + {filteredOptions.map((option) => ( + !isMultiSelect && handleOptionChange(option)} + isSelected={selectedValues.includes(option.value)} + isMultiSelect={isMultiSelect} + isDisabled={disabledValues?.includes(option.value)} + > + {isMultiSelect ? ( + + {option.label} + handleOptionChange(option)} + checked={selectedValues.includes(option.value)} + disabled={disabledValues?.includes(option.value)} + /> + + ) : ( + + + {option.label} + + {!!option.description && ( + + {option.description} + + )} + + )} + + ))} + + + )} + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Select/components.ts b/datahub-web-react/src/alchemy-components/components/Select/components.ts new file mode 100644 index 00000000000000..a360238fef4923 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/components.ts @@ -0,0 +1,235 @@ +import { Button, Icon } from '@components'; +import { borders, colors, radius, shadows, spacing, transition, typography } from '@components/theme'; +import { Checkbox } from 'antd'; +import styled from 'styled-components'; +import { formLabelTextStyles, inputPlaceholderTextStyles, inputValueTextStyles } from '../commonStyles'; +import { SelectSizeOptions, SelectStyleProps } from './types'; +import { getOptionLabelStyle, getSelectFontStyles, getSelectStyle } from './utils'; + +const sharedTransition = `${transition.property.colors} ${transition.easing['ease-in-out']} ${transition.duration.normal}`; + +/** + * Base Select component styling + */ +export const SelectBase = styled.div(({ isDisabled, isReadOnly, fontSize, isOpen }) => ({ + ...getSelectStyle({ isDisabled, isReadOnly, fontSize, isOpen }), + display: 'flex', + flexDirection: 'row' as const, + gap: spacing.xsm, + transition: sharedTransition, + justifyContent: 'space-between', + alignItems: 'center', + overflow: 'auto', + backgroundColor: isDisabled ? colors.gray[100] : 'white', +})); + +/** + * Styled components specific to the Basic version of the Select component + */ + +// Container for the Basic Select component +interface ContainerProps { + size: SelectSizeOptions; + width?: number | 'full'; +} + +export const Container = styled.div(({ size, width }) => ({ + position: 'relative', + display: 'flex', + flexDirection: 'column', + width: width === 'full' ? '100%' : `${width}px`, + gap: '4px', + transition: sharedTransition, + minWidth: '175px', + ...getSelectFontStyles(size), + ...inputValueTextStyles(size), +})); + +export const Dropdown = styled.div({ + position: 'absolute', + top: '100%', + left: 0, + right: 0, + borderRadius: radius.md, + background: colors.white, + zIndex: 1, + transition: sharedTransition, + boxShadow: shadows.dropdown, + padding: spacing.xsm, + display: 'flex', + flexDirection: 'column', + gap: '8px', + marginTop: '4px', + maxHeight: '360px', + overflow: 'auto', +}); + +export const SearchInputContainer = styled.div({ + position: 'relative', + width: '100%', + display: 'flex', + justifyContent: 'center', +}); + +export const SearchInput = styled.input({ + width: '100%', + borderRadius: radius.md, + border: `1px solid ${colors.gray[200]}`, + color: colors.gray[500], + fontFamily: typography.fonts.body, + fontSize: typography.fontSizes.sm, + padding: spacing.xsm, + paddingRight: spacing.xlg, + + '&:focus': { + borderColor: colors.violet[200], + outline: `${borders['1px']} ${colors.violet[200]}`, + }, +}); + +export const SearchIcon = styled(Icon)({ + position: 'absolute', + right: spacing.sm, + top: '50%', + transform: 'translateY(-50%)', + pointerEvents: 'none', +}); + +// Styled components for SelectValue (Selected value display) +export const SelectValue = styled.span({ + ...inputValueTextStyles(), +}); + +export const Placeholder = styled.span({ + ...inputPlaceholderTextStyles, +}); + +export const ActionButtonsContainer = styled.div({ + display: 'flex', + gap: '6px', + flexDirection: 'row', + alignItems: 'center', +}); + +/** + * Components that can be reused to create new Select variants + */ + +export const FooterBase = styled.div({ + display: 'flex', + justifyContent: 'flex-end', + gap: spacing.sm, + paddingTop: spacing.sm, + borderTop: `1px solid ${colors.gray[100]}`, +}); + +export const OptionList = styled.div({ + display: 'flex', + flexDirection: 'column' as const, +}); + +export const LabelContainer = styled.div({ + display: 'flex', + justifyContent: 'space-between', + width: '100%', +}); + +export const OptionContainer = styled.div({ + display: 'flex', + flexDirection: 'column', +}); + +export const DescriptionContainer = styled.span({ + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + width: '100%', + color: colors.gray[500], + lineHeight: 'normal', + fontSize: typography.fontSizes.sm, + marginTop: spacing.xxsm, +}); + +export const LabelsWrapper = styled.div({ + display: 'flex', + flexWrap: 'wrap', + gap: spacing.xxsm, + maxHeight: '150px', + maxWidth: 'calc(100% - 54px)', +}); + +export const OptionLabel = styled.label<{ isSelected: boolean; isMultiSelect?: boolean; isDisabled?: boolean }>( + ({ isSelected, isMultiSelect, isDisabled }) => ({ + ...getOptionLabelStyle(isSelected, isMultiSelect, isDisabled), + }), +); + +export const SelectAllOption = styled.div<{ isSelected: boolean; isDisabled?: boolean }>( + ({ isSelected, isDisabled }) => ({ + cursor: isDisabled ? 'not-allowed' : 'pointer', + padding: spacing.xsm, + color: isSelected ? colors.violet[700] : colors.gray[500], + fontWeight: typography.fontWeights.semiBold, + fontSize: typography.fontSizes.md, + display: 'flex', + alignItems: 'center', + }), +); + +export const SelectLabel = styled.label({ + ...formLabelTextStyles, + marginBottom: spacing.xxsm, + textAlign: 'left', +}); + +export const StyledCancelButton = styled(Button)({ + backgroundColor: colors.violet[100], + color: colors.violet[500], + borderColor: colors.violet[100], + + '&:hover': { + backgroundColor: colors.violet[200], + borderColor: colors.violet[200], + }, +}); + +export const StyledClearButton = styled(Button)({ + backgroundColor: colors.gray[200], + border: `1px solid ${colors.gray[200]}`, + color: colors.black, + padding: '1px', + + '&:hover': { + backgroundColor: colors.violet[100], + color: colors.violet[700], + borderColor: colors.violet[100], + boxShadow: shadows.none, + }, + + '&:focus': { + backgroundColor: colors.violet[100], + color: colors.violet[700], + boxShadow: `0 0 0 2px ${colors.white}, 0 0 0 4px ${colors.violet[50]}`, + }, +}); + +export const ClearIcon = styled.span({ + cursor: 'pointer', + marginLeft: '8px', +}); + +export const ArrowIcon = styled.span<{ isOpen: boolean }>(({ isOpen }) => ({ + marginLeft: 'auto', + border: 'solid black', + borderWidth: '0 1px 1px 0', + display: 'inline-block', + padding: '3px', + transform: isOpen ? 'rotate(-135deg)' : 'rotate(45deg)', +})); + +export const StyledCheckbox = styled(Checkbox)({ + '.ant-checkbox-checked:not(.ant-checkbox-disabled) .ant-checkbox-inner': { + backgroundColor: colors.violet[500], + borderColor: `${colors.violet[500]} !important`, + }, +}); diff --git a/datahub-web-react/src/alchemy-components/components/Select/index.ts b/datahub-web-react/src/alchemy-components/components/Select/index.ts new file mode 100644 index 00000000000000..eb469d0edc0046 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/index.ts @@ -0,0 +1,3 @@ +export { Select, selectDefaults } from './Select'; +export { SimpleSelect } from './SimpleSelect'; +export type { SelectProps, SelectOption } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Select/types.ts b/datahub-web-react/src/alchemy-components/components/Select/types.ts new file mode 100644 index 00000000000000..5ccde408b76999 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/types.ts @@ -0,0 +1,61 @@ +export type SelectSizeOptions = 'sm' | 'md' | 'lg'; + +export interface SelectOption { + value: string; + label: string; + description?: string; +} + +export interface SelectProps { + options: SelectOption[]; + label?: string; + values?: string[]; + onCancel?: () => void; + onUpdate?: (selectedValues: string[]) => void; + size?: SelectSizeOptions; + showSearch?: boolean; + isDisabled?: boolean; + isReadOnly?: boolean; + isRequired?: boolean; + showClear?: boolean; + width?: number | 'full'; + isMultiSelect?: boolean; + placeholder?: string; + disabledValues?: string[]; + showSelectAll?: boolean; + selectAllLabel?: string; + optionListTestId?: string; + showDescriptions?: boolean; +} + +export interface SelectStyleProps { + fontSize?: SelectSizeOptions; + isDisabled?: boolean; + isReadOnly?: boolean; + isRequired?: boolean; + isOpen?: boolean; +} + +export interface ActionButtonsProps { + fontSize?: SelectSizeOptions; + selectedValues: string[]; + isOpen: boolean; + isDisabled: boolean; + isReadOnly: boolean; + showClear: boolean; + handleClearSelection: () => void; +} + +export interface SelectLabelDisplayProps { + selectedValues: string[]; + options: SelectOption[]; + placeholder: string; + isMultiSelect?: boolean; + removeOption?: (option: SelectOption) => void; + disabledValues?: string[]; + showDescriptions?: boolean; +} + +export interface SearchInputProps extends React.InputHTMLAttributes { + fontSize: SelectSizeOptions; +} diff --git a/datahub-web-react/src/alchemy-components/components/Select/utils.ts b/datahub-web-react/src/alchemy-components/components/Select/utils.ts new file mode 100644 index 00000000000000..d054dd8ff737ad --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Select/utils.ts @@ -0,0 +1,125 @@ +import { borders, colors, radius, spacing, typography } from '@components/theme'; +import { getFontSize } from '@components/theme/utils'; + +import { SelectStyleProps } from './types'; + +export const getOptionLabelStyle = (isSelected: boolean, isMultiSelect?: boolean, isDisabled?: boolean) => ({ + cursor: isDisabled ? 'not-allowed' : 'pointer', + padding: spacing.xsm, + borderRadius: radius.md, + lineHeight: typography.lineHeights.normal, + backgroundColor: isSelected && !isMultiSelect ? colors.violet[100] : 'transparent', + color: isSelected ? colors.violet[700] : colors.gray[500], + fontWeight: typography.fontWeights.medium, + fontSize: typography.fontSizes.md, + display: 'flex', + alignItems: 'center', + + '&:hover': { + backgroundColor: isSelected ? colors.violet[100] : colors.gray[100], + }, +}); + +export const getFooterButtonSize = (size) => { + return size === 'sm' ? 'sm' : 'md'; +}; + +export const getSelectFontStyles = (size) => { + const baseFontStyles = { + lineHeight: typography.lineHeights.none, + }; + + const sizeStyles = { + sm: { + ...baseFontStyles, + fontSize: getFontSize(size), + }, + md: { + ...baseFontStyles, + fontSize: getFontSize(size), + }, + lg: { + ...baseFontStyles, + fontSize: getFontSize(size), + }, + }; + + return sizeStyles[size]; +}; + +export const getSelectPadding = (size) => { + const paddingStyles = { + sm: { + padding: `${spacing.sm} ${spacing.xsm}`, + }, + md: { + padding: `${spacing.sm} ${spacing.md}`, + }, + lg: { + padding: `${spacing.md} ${spacing.sm}`, + }, + }; + + return paddingStyles[size]; +}; + +export const getSearchPadding = (size) => { + const paddingStyles = { + sm: { + padding: `${spacing.xxsm} ${spacing.xsm}`, + }, + md: { + padding: `${spacing.xsm} ${spacing.xsm}`, + }, + lg: { + padding: `${spacing.xsm} ${spacing.xsm}`, + }, + }; + + return paddingStyles[size]; +}; + +export const getSelectStyle = (props: SelectStyleProps) => { + const { isDisabled, isReadOnly, fontSize, isOpen } = props; + + const baseStyle = { + borderRadius: radius.md, + border: `1px solid ${colors.gray[200]}`, + fontFamily: typography.fonts.body, + color: isDisabled ? colors.gray[300] : colors.black, + cursor: isDisabled || isReadOnly ? 'not-allowed' : 'pointer', + backgroundColor: isDisabled ? colors.gray[100] : 'initial', + + '&::placeholder': { + color: colors.gray[400], + }, + + // Open Styles + ...(isOpen + ? { + borderColor: colors.violet[300], + boxShadow: `0px 0px 4px 0px rgba(83, 63, 209, 0.5)`, + outline: 'none', + } + : {}), + + // Hover Styles + ...(isDisabled || isReadOnly || isOpen + ? {} + : { + '&:hover': { + borderColor: colors.violet[200], + outline: `${borders['1px']} ${colors.violet[200]}`, + }, + }), + }; + + const fontStyles = getSelectFontStyles(fontSize); + const paddingStyles = getSelectPadding(fontSize); + + return { + ...baseStyle, + ...fontStyles, + ...paddingStyles, + }; +}; diff --git a/datahub-web-react/src/alchemy-components/components/Switch/Switch.stories.tsx b/datahub-web-react/src/alchemy-components/components/Switch/Switch.stories.tsx new file mode 100644 index 00000000000000..7bb4ee2397cc63 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Switch/Switch.stories.tsx @@ -0,0 +1,169 @@ +import React from 'react'; + +import type { Meta, StoryObj } from '@storybook/react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; + +import { GridList } from '@components/.docs/mdx-components'; + +import { Switch, switchDefaults } from './Switch'; +import { AVAILABLE_ICONS } from '../Icon'; + +const meta = { + title: 'Forms / Switch', + component: Switch, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'A component that is used to get user input in the state of a toggle.', + }, + }, + + // Component-level argTypes + argTypes: { + label: { + description: 'Label for the Switch.', + table: { + defaultValue: { summary: switchDefaults.label }, + }, + control: { + type: 'text', + }, + }, + labelPosition: { + description: 'The position of the label relative to the Switch.', + options: ['left', 'top'], + table: { + defaultValue: { summary: switchDefaults.labelPosition }, + }, + control: { + type: 'select', + }, + }, + icon: { + description: 'The icon to display in the Switch Slider.', + type: 'string', + options: AVAILABLE_ICONS, + table: { + defaultValue: { summary: 'undefined' }, + }, + control: { + type: 'select', + }, + }, + colorScheme: { + description: 'The color of the Switch.', + options: ['violet', 'green', 'red', 'blue', 'gray'], + table: { + defaultValue: { summary: switchDefaults.colorScheme }, + }, + control: { + type: 'select', + }, + }, + size: { + description: 'The size of the Button.', + options: ['sm', 'md', 'lg', 'xl'], + table: { + defaultValue: { summary: switchDefaults.size }, + }, + control: { + type: 'select', + }, + }, + isSquare: { + description: 'Whether the Switch is square in shape.', + table: { + defaultValue: { summary: switchDefaults?.isSquare?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isChecked: { + description: 'Whether the Switch is checked.', + table: { + defaultValue: { summary: switchDefaults?.isChecked?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isDisabled: { + description: 'Whether the Switch is in disabled state.', + table: { + defaultValue: { summary: switchDefaults?.isDisabled?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isRequired: { + description: 'Whether the Switch is a required field.', + table: { + defaultValue: { summary: switchDefaults?.isRequired?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + }, + + // Define defaults + args: { + label: switchDefaults.label, + labelPosition: switchDefaults.labelPosition, + icon: switchDefaults.icon, + colorScheme: switchDefaults.colorScheme, + size: switchDefaults.size, + isSquare: switchDefaults.isSquare, + isChecked: switchDefaults.isChecked, + isDisabled: switchDefaults.isDisabled, + isRequired: switchDefaults.isRequired, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const sizes = () => ( + + + + + + +); + +export const colors = () => ( + + + + + + + +); + +export const states = () => ( + + + + + +); + +export const types = () => ( + + + + +); diff --git a/datahub-web-react/src/alchemy-components/components/Switch/Switch.tsx b/datahub-web-react/src/alchemy-components/components/Switch/Switch.tsx new file mode 100644 index 00000000000000..18a01386562ee9 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Switch/Switch.tsx @@ -0,0 +1,74 @@ +import { Tooltip } from '@components'; +import React, { useEffect, useState } from 'react'; +import { IconContainer, Label, Required, Slider, StyledIcon, StyledInput, SwitchContainer } from './components'; +import { SwitchProps } from './types'; + +export const switchDefaults: SwitchProps = { + label: 'Label', + labelPosition: 'left', + colorScheme: 'violet', + size: 'md', + isSquare: false, + isChecked: false, + isDisabled: false, + isRequired: false, +}; + +export const Switch = ({ + label = switchDefaults.label, + labelPosition = switchDefaults.labelPosition, + icon, // undefined by default + colorScheme = switchDefaults.colorScheme, + size = switchDefaults.size, + isSquare = switchDefaults.isSquare, + isChecked = switchDefaults.isChecked, + isDisabled = switchDefaults.isDisabled, + isRequired = switchDefaults.isRequired, + labelHoverText, + disabledHoverText, + labelStyle, + ...props +}: SwitchProps) => { + const [checked, setChecked] = useState(isChecked); + + useEffect(() => { + setChecked(isChecked); + }, [isChecked]); + + const id = props.id || `switchToggle-${label}`; + + return ( + + + + + setChecked(!checked)} + customSize={size} + disabled={isDisabled} + colorScheme={colorScheme || 'violet'} + aria-labelledby={id} + aria-checked={checked} + {...props} + /> + + + + {icon && ( + + )} + + + + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Switch/components.ts b/datahub-web-react/src/alchemy-components/components/Switch/components.ts new file mode 100644 index 00000000000000..1586c1cf9f32fd --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Switch/components.ts @@ -0,0 +1,118 @@ +import styled from 'styled-components'; + +import { borders, colors, shadows, spacing, transition } from '@components/theme'; +import { ColorOptions, SizeOptions } from '@components/theme/config'; + +import { Icon } from '../Icon'; + +import { formLabelTextStyles } from '../commonStyles'; + +import { + getIconTransformPositionLeft, + getIconTransformPositionTop, + getInputHeight, + getSliderTransformPosition, + getToggleSize, +} from './utils'; + +import type { SwitchLabelPosition } from './types'; + +export const Label = styled.div({ + ...formLabelTextStyles, + display: 'flex', + alignItems: 'flex-start', +}); + +export const SwitchContainer = styled.label<{ labelPosition: SwitchLabelPosition; isDisabled?: boolean }>( + ({ labelPosition, isDisabled }) => ({ + display: 'flex', + flexDirection: labelPosition === 'top' ? 'column' : 'row', + alignItems: labelPosition === 'top' ? 'flex-start' : 'center', + gap: spacing.sm, + cursor: isDisabled ? 'not-allowed' : 'pointer', + width: 'max-content', + }), +); + +export const Slider = styled.div<{ size?: SizeOptions; isSquare?: boolean; isDisabled?: boolean }>( + ({ size, isSquare, isDisabled }) => ({ + '&:before': { + transition: `${transition.duration.normal} all`, + content: '""', + position: 'absolute', + minWidth: getToggleSize(size || 'md', 'slider'), // sliders width and height must be same + minHeight: getToggleSize(size || 'md', 'slider'), + borderRadius: !isSquare ? '35px' : '0px', + top: '50%', + left: spacing.xxsm, + transform: 'translate(0, -50%)', + backgroundColor: !isDisabled ? colors.white : colors.gray[200], + boxShadow: ` + 0px 1px 2px 0px rgba(16, 24, 40, 0.06), + 0px 1px 3px 0px rgba(16, 24, 40, 0.12) + `, + }, + borderRadius: !isSquare ? '32px' : '0px', + minWidth: getToggleSize(size || 'md', 'input'), + minHeight: getInputHeight(size || 'md'), + }), + { + display: 'flex', + justifyContent: 'center', + alignItems: 'center', + position: 'relative', + + backgroundColor: colors.gray[100], + padding: spacing.xxsm, + transition: `${transition.duration.normal} all`, + boxSizing: 'content-box', + }, +); + +export const Required = styled.span({ + color: colors.red[500], + marginLeft: spacing.xxsm, +}); + +export const StyledInput = styled.input<{ + customSize?: SizeOptions; + disabled?: boolean; + colorScheme: ColorOptions; + checked?: boolean; +}>` + opacity: 0; + position: absolute; + + &:checked + ${Slider} { + background-color: ${(props) => (!props.disabled ? colors[props.colorScheme][500] : colors.gray[100])}; + + &:before { + transform: ${({ customSize }) => getSliderTransformPosition(customSize || 'md')}; + } + } + + &:focus-within + ${Slider} { + border-color: ${(props) => (props.checked ? colors[props.colorScheme][200] : 'transparent')}; + outline: ${(props) => (props.checked ? `${borders['2px']} ${colors[props.colorScheme][200]}` : 'none')}; + box-shadow: ${(props) => (props.checked ? shadows.xs : 'none')}; + } +`; + +export const StyledIcon = styled(Icon)<{ checked?: boolean; size: SizeOptions }>( + ({ checked, size }) => ({ + left: getIconTransformPositionLeft(size, checked || false), + top: getIconTransformPositionTop(size), + }), + { + transition: `${transition.duration.normal} all`, + position: 'absolute', + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + color: colors.gray[500], + }, +); + +export const IconContainer = styled.div({ + position: 'relative', +}); diff --git a/datahub-web-react/src/alchemy-components/components/Switch/index.ts b/datahub-web-react/src/alchemy-components/components/Switch/index.ts new file mode 100644 index 00000000000000..0c48d2964887ec --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Switch/index.ts @@ -0,0 +1,2 @@ +export { Switch, switchDefaults } from './Switch'; +export type { SwitchProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Switch/types.ts b/datahub-web-react/src/alchemy-components/components/Switch/types.ts new file mode 100644 index 00000000000000..e15c0f81b4a392 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Switch/types.ts @@ -0,0 +1,21 @@ +import { ColorOptions, SizeOptions } from '@components/theme/config'; +import { InputHTMLAttributes } from 'react'; +import { CSSProperties } from 'styled-components'; +import { IconNames } from '../Icon'; + +export type SwitchLabelPosition = 'left' | 'top'; + +export interface SwitchProps extends Omit, 'size'> { + label: string; + labelPosition?: SwitchLabelPosition; + icon?: IconNames; + colorScheme?: ColorOptions; + size?: SizeOptions; + isSquare?: boolean; + isChecked?: boolean; + isDisabled?: boolean; + isRequired?: boolean; + labelHoverText?: string; + disabledHoverText?: string; + labelStyle?: CSSProperties; +} diff --git a/datahub-web-react/src/alchemy-components/components/Switch/utils.ts b/datahub-web-react/src/alchemy-components/components/Switch/utils.ts new file mode 100644 index 00000000000000..c0365baa348183 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Switch/utils.ts @@ -0,0 +1,97 @@ +import { SizeOptions } from '@components/theme/config'; + +const sliderSize = { + sm: '14px', + md: '16px', + lg: '18px', + xl: '20px', +}; + +const inputSize = { + sm: '35px', + md: '40px', + lg: '45px', + xl: '50px', +}; + +const translateSize = { + sm: '22px', + md: '24px', + lg: '26px', + xl: '28px', +}; + +const iconTransformPositionLeft = { + sm: { + checked: '5.5px', + unchecked: '-16.5px', + }, + md: { + checked: '5px', + unchecked: '-19px', + }, + lg: { + checked: '4.5px', + unchecked: '-21.5px', + }, + xl: { + checked: '4px', + unchecked: '-24px', + }, +}; + +const iconTransformPositionTop = { + sm: '-6px', + md: '-7px', + lg: '-8px', + xl: '-9px', +}; + +export const getToggleSize = (size: SizeOptions, mode: 'slider' | 'input'): string => { + if (size === 'sm') return mode === 'slider' ? sliderSize.sm : inputSize.sm; + if (size === 'md') return mode === 'slider' ? sliderSize.md : inputSize.md; + if (size === 'lg') return mode === 'slider' ? sliderSize.lg : inputSize.lg; + return mode === 'slider' ? sliderSize.xl : inputSize.xl; // xl +}; + +export const getInputHeight = (size: SizeOptions) => { + if (size === 'sm') return sliderSize.sm; + if (size === 'md') return sliderSize.md; + if (size === 'lg') return sliderSize.lg; + return sliderSize.xl; // xl +}; + +export const getSliderTransformPosition = (size: SizeOptions): string => { + if (size === 'sm') return `translate(${translateSize.sm}, -50%)`; + if (size === 'md') return `translate(${translateSize.md}, -50%)`; + if (size === 'lg') return `translate(${translateSize.lg}, -50%)`; + return `translate(${translateSize.xl}, -50%)`; // xl +}; + +export const getIconTransformPositionLeft = (size: SizeOptions, checked: boolean): string => { + if (size === 'sm') { + if (checked) return iconTransformPositionLeft.sm.checked; + return iconTransformPositionLeft.sm.unchecked; + } + + if (size === 'md') { + if (checked) return iconTransformPositionLeft.md.checked; + return iconTransformPositionLeft.md.unchecked; + } + + if (size === 'lg') { + if (checked) return iconTransformPositionLeft.lg.checked; + return iconTransformPositionLeft.lg.unchecked; + } + + // xl + if (checked) return iconTransformPositionLeft.xl.checked; + return iconTransformPositionLeft.xl.unchecked; +}; + +export const getIconTransformPositionTop = (size: SizeOptions): string => { + if (size === 'sm') return iconTransformPositionTop.sm; + if (size === 'md') return iconTransformPositionTop.md; + if (size === 'lg') return iconTransformPositionTop.lg; + return iconTransformPositionTop.xl; // xl +}; diff --git a/datahub-web-react/src/alchemy-components/components/Table/Table.stories.tsx b/datahub-web-react/src/alchemy-components/components/Table/Table.stories.tsx new file mode 100644 index 00000000000000..3a36b658978066 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Table/Table.stories.tsx @@ -0,0 +1,162 @@ +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import { Meta, StoryObj } from '@storybook/react'; +import React from 'react'; +import { Table, tableDefaults } from '.'; + +// Auto Docs +const meta = { + title: 'Lists & Tables / Table', + component: Table, + + // Display Properties + parameters: { + layout: 'padded', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'This component allows users to render a table with different columns and their data', + }, + }, + + // Component-level argTypes + argTypes: { + columns: { + description: 'Array of column objects for the table header.', + control: 'object', + table: { + defaultValue: { summary: JSON.stringify(tableDefaults.columns) }, + }, + }, + data: { + description: 'Array of data rows for the table body.', + control: 'object', + table: { + defaultValue: { summary: JSON.stringify(tableDefaults.data) }, + }, + }, + showHeader: { + description: 'Whether to show the table header.', + control: 'boolean', + table: { + defaultValue: { summary: tableDefaults.showHeader?.toString() }, + }, + }, + isLoading: { + description: 'Whether the table is in loading state.', + control: 'boolean', + table: { + defaultValue: { summary: tableDefaults.isLoading?.toString() }, + }, + }, + isScrollable: { + description: 'Whether the table is scrollable.', + control: 'boolean', + table: { + defaultValue: { summary: tableDefaults.isScrollable?.toString() }, + }, + }, + maxHeight: { + description: 'Maximum height of the table container.', + control: 'text', + table: { + defaultValue: { summary: tableDefaults.maxHeight }, + }, + }, + }, + + // Define defaults + args: { + columns: [ + { title: 'Column 1', key: 'column1', dataIndex: 'column1' }, + { title: 'Column 2', key: 'column2', dataIndex: 'column2' }, + ], + data: [ + { column1: 'Row 1 Col 1', column2: 'Row 1 Col 2' }, + { column1: 'Row 2 Col 1', column2: 'Row 2 Col 2' }, + ], + showHeader: tableDefaults.showHeader, + isLoading: tableDefaults.isLoading, + isScrollable: tableDefaults.isScrollable, + maxHeight: tableDefaults.maxHeight, + }, +} satisfies Meta>; + +export default meta; + +// Stories + +type Story = StoryObj; + +// Basic story is what is displayed 1st in storybook & is used as the code sandbox +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => , +}; + +export const withScroll = () => ( +
+); + +export const withCustomColumnWidths = () => ( +
+); + +export const withColumnSorting = () => ( +
a.column1.localeCompare(b.column1), + }, + { title: 'Column 2', key: 'column2', dataIndex: 'column2' }, + { title: 'Column 3', key: 'column3', dataIndex: 'column3', sorter: (a, b) => a.column3 - b.column3 }, + ]} + data={[ + { column1: 'Row 2 Col 1', column2: 'Row 2 Col 2', column3: 3 }, + { column1: 'Row 1 Col 1', column2: 'Row 1 Col 2', column3: 2 }, + { column1: 'Row 3 Col 1', column2: 'Row 3 Col 2', column3: 1 }, + ]} + /> +); + +export const withoutHeader = () => ( +
+); diff --git a/datahub-web-react/src/alchemy-components/components/Table/Table.tsx b/datahub-web-react/src/alchemy-components/components/Table/Table.tsx new file mode 100644 index 00000000000000..11e598f8d4e0f7 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Table/Table.tsx @@ -0,0 +1,115 @@ +import { LoadingOutlined } from '@ant-design/icons'; +import { Text } from '@components'; +import React, { useState } from 'react'; +import { + BaseTable, + HeaderContainer, + LoadingContainer, + SortIcon, + SortIconsContainer, + TableCell, + TableContainer, + TableHeader, + TableHeaderCell, + TableRow, +} from './components'; +import { TableProps } from './types'; +import { getSortedData, handleActiveSort, renderCell, SortingState } from './utils'; + +export const tableDefaults: TableProps = { + columns: [], + data: [], + showHeader: true, + isLoading: false, + isScrollable: false, + maxHeight: '100%', +}; + +export const Table = ({ + columns = tableDefaults.columns, + data = tableDefaults.data, + showHeader = tableDefaults.showHeader, + isLoading = tableDefaults.isLoading, + isScrollable = tableDefaults.isScrollable, + maxHeight = tableDefaults.maxHeight, + ...props +}: TableProps) => { + const [sortColumn, setSortColumn] = useState(null); + const [sortOrder, setSortOrder] = useState(SortingState.ORIGINAL); + + const sortedData = getSortedData(columns, data, sortColumn, sortOrder); + + if (isLoading) { + return ( + + + Loading data... + + ); + } + + return ( + + + {showHeader && ( + + + {columns.map((column) => ( + + + {column.title} + {column.sorter && ( + + column.sorter && + handleActiveSort( + column.key, + sortColumn, + setSortColumn, + setSortOrder, + ) + } + > + + + + )} + + + ))} + + + )} + + {sortedData.map((row, index) => ( + + {columns.map((column) => { + return ( + + {renderCell(column, row, index)} + + ); + })} + + ))} + + + + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/Table/components.ts b/datahub-web-react/src/alchemy-components/components/Table/components.ts new file mode 100644 index 00000000000000..8908256a81ddf2 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Table/components.ts @@ -0,0 +1,94 @@ +import { Icon } from '@components'; +import { colors, radius, spacing, typography } from '@src/alchemy-components/theme'; +import { AlignmentOptions } from '@src/alchemy-components/theme/config'; +import styled from 'styled-components'; + +export const TableContainer = styled.div<{ isScrollable?: boolean; maxHeight?: string }>( + ({ isScrollable, maxHeight }) => ({ + borderRadius: radius.lg, + border: `1px solid ${colors.gray[1400]}`, + overflow: isScrollable ? 'auto' : 'hidden', + width: '100%', + maxHeight: maxHeight || '100%', + }), +); + +export const BaseTable = styled.table({ + borderCollapse: 'collapse', + width: '100%', +}); + +export const TableHeader = styled.thead({ + backgroundColor: colors.gray[1500], + borderRadius: radius.lg, + position: 'sticky', + top: 0, + zIndex: 100, +}); + +export const TableHeaderCell = styled.th<{ width?: string }>(({ width }) => ({ + padding: `${spacing.sm} ${spacing.md}`, + color: colors.gray[600], + fontSize: typography.fontSizes.sm, + fontWeight: typography.fontWeights.medium, + textAlign: 'start', + width: width || 'auto', +})); + +export const HeaderContainer = styled.div({ + display: 'flex', + alignItems: 'center', + gap: spacing.sm, +}); + +export const TableRow = styled.tr({ + '&:last-child': { + '& td': { + borderBottom: 'none', + }, + }, + + '& td:first-child': { + fontWeight: typography.fontWeights.medium, + color: colors.gray[600], + }, +}); + +export const TableCell = styled.td<{ width?: string; alignment?: AlignmentOptions }>(({ width, alignment }) => ({ + padding: spacing.md, + borderBottom: `1px solid ${colors.gray[1400]}`, + color: colors.gray[1700], + fontSize: typography.fontSizes.md, + fontWeight: typography.fontWeights.normal, + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + maxWidth: width || 'unset', + textAlign: alignment || 'left', +})); + +export const SortIconsContainer = styled.div({ + display: 'flex', + flexDirection: 'column', +}); + +export const SortIcon = styled(Icon)<{ isActive?: boolean }>(({ isActive }) => ({ + margin: '-3px', + stroke: isActive ? colors.violet[600] : undefined, + + ':hover': { + cursor: 'pointer', + }, +})); + +export const LoadingContainer = styled.div({ + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + height: '100%', + width: '100%', + gap: spacing.sm, + color: colors.violet[700], + fontSize: typography.fontSizes['3xl'], +}); diff --git a/datahub-web-react/src/alchemy-components/components/Table/index.ts b/datahub-web-react/src/alchemy-components/components/Table/index.ts new file mode 100644 index 00000000000000..986f467da74b8c --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Table/index.ts @@ -0,0 +1,2 @@ +export { Table, tableDefaults } from './Table'; +export type { Column, TableProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Table/types.ts b/datahub-web-react/src/alchemy-components/components/Table/types.ts new file mode 100644 index 00000000000000..b3e0357d5cf147 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Table/types.ts @@ -0,0 +1,21 @@ +import { AlignmentOptions } from '@src/alchemy-components/theme/config'; +import { TableHTMLAttributes } from 'react'; + +export interface Column { + title: string; + key: string; + dataIndex?: string; + render?: (record: T, index: number) => React.ReactNode; + width?: string; + sorter?: (a: T, b: T) => number; + alignment?: AlignmentOptions; +} + +export interface TableProps extends TableHTMLAttributes { + columns: Column[]; + data: T[]; + showHeader?: boolean; + isLoading?: boolean; + isScrollable?: boolean; + maxHeight?: string; +} diff --git a/datahub-web-react/src/alchemy-components/components/Table/utils.ts b/datahub-web-react/src/alchemy-components/components/Table/utils.ts new file mode 100644 index 00000000000000..c76494d32ca633 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Table/utils.ts @@ -0,0 +1,73 @@ +import { Column } from './types'; + +export enum SortingState { + ASCENDING = 'ascending', + DESCENDING = 'descending', + ORIGINAL = 'original', +} + +export const handleActiveSort = ( + key: string, + sortColumn: string | null, + setSortColumn: React.Dispatch>, + setSortOrder: React.Dispatch>, +) => { + if (sortColumn === key) { + // Toggle sort order + setSortOrder((prevOrder) => { + if (prevOrder === SortingState.ASCENDING) return SortingState.DESCENDING; + if (prevOrder === SortingState.DESCENDING) return SortingState.ORIGINAL; + return SortingState.ASCENDING; + }); + } else { + // Set new column and default sort order + setSortColumn(key); + setSortOrder(SortingState.ASCENDING); + } +}; + +export const getSortedData = ( + columns: Column[], + data: T[], + sortColumn: string | null, + sortOrder: SortingState, +) => { + if (sortOrder === SortingState.ORIGINAL || !sortColumn) { + return data; + } + + const activeColumn = columns.find((column) => column.key === sortColumn); + + // Sort based on the order and column sorter + if (activeColumn && activeColumn.sorter) { + return data.slice().sort((a, b) => { + return sortOrder === SortingState.ASCENDING ? activeColumn.sorter!(a, b) : activeColumn.sorter!(b, a); + }); + } + + return data; +}; + +export const renderCell = (column: Column, row: T, index: number) => { + const { render, dataIndex } = column; + + let cellData; + + if (dataIndex) { + cellData = row[dataIndex]; + + if (typeof dataIndex === 'string') { + cellData = dataIndex.split('.').reduce((acc, prop) => acc && acc[prop], row); + } + + if (Array.isArray(dataIndex)) { + cellData = dataIndex.reduce((acc, prop) => acc && acc[prop], row); + } + } + + if (render) { + return render(row, index); + } + + return cellData; +}; diff --git a/datahub-web-react/src/alchemy-components/components/Text/Text.stories.tsx b/datahub-web-react/src/alchemy-components/components/Text/Text.stories.tsx new file mode 100644 index 00000000000000..c82d468aaa08ce --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Text/Text.stories.tsx @@ -0,0 +1,100 @@ +import React from 'react'; + +import type { Meta, StoryObj, StoryFn } from '@storybook/react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; + +import { VerticalFlexGrid } from '@components/.docs/mdx-components'; +import { Text, textDefaults } from '.'; + +// Auto Docs +const meta = { + title: 'Typography / Text', + component: Text, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'Used to render text and paragraphs within an interface.', + }, + }, + + // Component-level argTypes + argTypes: { + children: { + description: 'The content to display within the heading.', + table: { + type: { summary: 'string' }, + }, + }, + type: { + description: 'The type of text to display.', + table: { + defaultValue: { summary: textDefaults.type }, + }, + }, + size: { + description: 'Override the size of the text.', + table: { + defaultValue: { summary: `${textDefaults.size}` }, + }, + }, + color: { + description: 'Override the color of the text.', + table: { + defaultValue: { summary: textDefaults.color }, + }, + }, + weight: { + description: 'Override the weight of the heading.', + table: { + defaultValue: { summary: textDefaults.weight }, + }, + }, + }, + + // Define default args + args: { + children: + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas aliquet nulla id felis vehicula, et posuere dui dapibus. Nullam rhoncus massa non tortor convallis, in blandit turpis rutrum. Morbi tempus velit mauris, at mattis metus mattis sed. Nunc molestie efficitur lectus, vel mollis eros.', + type: textDefaults.type, + size: textDefaults.size, + color: textDefaults.color, + weight: textDefaults.weight, + }, +} satisfies Meta; + +export default meta; + +// Stories + +type Story = StoryObj; + +// Basic story is what is displayed 1st in storybook +// Pass props to this so that it can be customized via the UI props panel +export const sandbox: Story = { + tags: ['dev'], + render: (props) => {props.children}, +}; + +export const sizes: StoryFn = (props: any) => ( + + {props.children} + {props.children} + {props.children} + {props.children} + {props.children} + {props.children} + {props.children} + {props.children} + +); + +export const withLink = () => ( + + Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas aliquet nulla id felis vehicula, et posuere + dui dapibus. Nullam rhoncus massa non tortor convallis, in blandit turpis rutrum. Morbi tempus + velit mauris, at mattis metus mattis sed. Nunc molestie efficitur lectus, vel mollis eros. + +); diff --git a/datahub-web-react/src/alchemy-components/components/Text/Text.tsx b/datahub-web-react/src/alchemy-components/components/Text/Text.tsx new file mode 100644 index 00000000000000..89122afbfcc8bf --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Text/Text.tsx @@ -0,0 +1,33 @@ +import React from 'react'; + +import { TextProps } from './types'; +import { P, Div, Span } from './components'; + +export const textDefaults: TextProps = { + type: 'p', + color: 'inherit', + size: 'md', + weight: 'normal', +}; + +export const Text = ({ + type = textDefaults.type, + color = textDefaults.color, + size = textDefaults.size, + weight = textDefaults.weight, + children, + ...props +}: TextProps) => { + const sharedProps = { size, color, weight, ...props }; + + switch (type) { + case 'p': + return

{children}

; + case 'div': + return
{children}
; + case 'span': + return {children}; + default: + return

{children}

; + } +}; diff --git a/datahub-web-react/src/alchemy-components/components/Text/components.ts b/datahub-web-react/src/alchemy-components/components/Text/components.ts new file mode 100644 index 00000000000000..1d48497f39c9c8 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Text/components.ts @@ -0,0 +1,50 @@ +import styled from 'styled-components'; + +import { typography, colors } from '@components/theme'; +import { getColor, getFontSize } from '@components/theme/utils'; +import { TextProps } from './types'; + +// Text Styles +const textStyles = { + fontSize: typography.fontSizes.md, + lineHeight: typography.lineHeights.md, + fontWeight: typography.fontWeights.normal, +}; + +// Default styles +const baseStyles = { + fontFamily: typography.fonts.body, + margin: 0, + + '& a': { + color: colors.violet[400], + textDecoration: 'none', + transition: 'color 0.15s ease', + + '&:hover': { + color: colors.violet[500], + }, + }, +}; + +// Prop Driven Styles +const propStyles = (props, isText = false) => { + const styles = {} as any; + if (props.size) styles.fontSize = getFontSize(props.size); + if (props.color) styles.color = getColor(props.color); + if (props.weight) styles.fontWeight = typography.fontWeights[props.weight]; + if (isText) styles.lineHeight = typography.lineHeights[props.size || 'md']; + return styles; +}; + +export const P = styled.p({ ...baseStyles, ...textStyles }, (props: TextProps) => ({ + ...propStyles(props as TextProps, true), +})); + +export const Span = styled.span({ ...baseStyles, ...textStyles }, (props: TextProps) => ({ + ...propStyles(props as TextProps, true), +})); + +export const Div = styled.div({ ...baseStyles, ...textStyles }, (props: TextProps) => ({ + ...propStyles(props as TextProps, true), +})); diff --git a/datahub-web-react/src/alchemy-components/components/Text/index.ts b/datahub-web-react/src/alchemy-components/components/Text/index.ts new file mode 100644 index 00000000000000..d4240105173d48 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Text/index.ts @@ -0,0 +1,2 @@ +export { Text, textDefaults } from './Text'; +export type { TextProps } from './types'; diff --git a/datahub-web-react/src/alchemy-components/components/Text/types.ts b/datahub-web-react/src/alchemy-components/components/Text/types.ts new file mode 100644 index 00000000000000..6a41929da12a9b --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/Text/types.ts @@ -0,0 +1,9 @@ +import { HTMLAttributes } from 'react'; +import type { FontSizeOptions, FontColorOptions, FontWeightOptions } from '@components/theme/config'; + +export interface TextProps extends HTMLAttributes { + type?: 'span' | 'p' | 'div'; + size?: FontSizeOptions; + color?: FontColorOptions; + weight?: FontWeightOptions; +} diff --git a/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.stories.tsx b/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.stories.tsx new file mode 100644 index 00000000000000..b244eefa6f2073 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/TextArea/TextArea.stories.tsx @@ -0,0 +1,159 @@ +import React from 'react'; + +import type { Meta, StoryObj } from '@storybook/react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; + +import { GridList } from '@components/.docs/mdx-components'; + +import { TextArea, textAreaDefaults } from './TextArea'; +import { AVAILABLE_ICONS } from '../Icon'; + +// Auto Docs +const meta = { + title: 'Forms / Text Area', + component: TextArea, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'A component that is used to get user input in a text area field.', + }, + }, + + // Component-level argTypes + argTypes: { + label: { + description: 'Label for the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults.label }, + }, + control: { + type: 'text', + }, + }, + placeholder: { + description: 'Placeholder for the Text Area.', + table: { + defaultValue: { summary: textAreaDefaults.placeholder }, + }, + control: { + type: 'text', + }, + }, + icon: { + description: 'The icon to display in the Text Area.', + type: 'string', + options: AVAILABLE_ICONS, + table: { + defaultValue: { summary: 'undefined' }, + }, + control: { + type: 'select', + }, + }, + error: { + description: 'Enforce error state on the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults.error }, + }, + control: { + type: 'text', + }, + }, + warning: { + description: 'Enforce warning state on the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults.warning }, + }, + control: { + type: 'text', + }, + }, + isSuccess: { + description: 'Enforce success state on the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults?.isSuccess?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isDisabled: { + description: 'Enforce disabled state on the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults?.isDisabled?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isInvalid: { + description: 'Enforce invalid state on the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults?.isInvalid?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isReadOnly: { + description: 'Enforce read only state on the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults?.isReadOnly?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + isRequired: { + description: 'Enforce required state on the TextArea.', + table: { + defaultValue: { summary: textAreaDefaults?.isRequired?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + }, + + // Define defaults + args: { + label: textAreaDefaults.label, + placeholder: textAreaDefaults.placeholder, + icon: textAreaDefaults.icon, + error: textAreaDefaults.error, + warning: textAreaDefaults.warning, + isSuccess: textAreaDefaults.isSuccess, + isDisabled: textAreaDefaults.isDisabled, + isInvalid: textAreaDefaults.isInvalid, + isReadOnly: textAreaDefaults.isReadOnly, + isRequired: textAreaDefaults.isRequired, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) =>