diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index 398adc3708ef2b..5cc51882965a0c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -499,6 +499,7 @@ def get_schema_fields_for_column( self, dataset_name: str, column: Dict, + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, @@ -506,6 +507,7 @@ def get_schema_fields_for_column( fields = get_schema_fields_for_sqlalchemy_column( column_name=column["name"], column_type=column["type"], + inspector=inspector, description=column.get("comment", None), nullable=column.get("nullable", True), is_part_of_key=( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py index 65f8516fd340a3..59f301baf40165 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive.py @@ -169,12 +169,16 @@ def get_schema_fields_for_column( self, dataset_name: str, column: Dict[Any, Any], + inspector: Inspector, pk_constraints: Optional[Dict[Any, Any]] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, ) -> List[SchemaField]: fields = super().get_schema_fields_for_column( - dataset_name, column, pk_constraints + dataset_name, + column, + inspector, + pk_constraints, ) if self._COMPLEX_TYPE.match(fields[0].nativeDataType) and isinstance( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py index 655d1ba68ed79e..9da6c294881247 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py @@ -521,7 +521,7 @@ def loop_tables( ) # add table schema fields - schema_fields = self.get_schema_fields(dataset_name, columns) + schema_fields = self.get_schema_fields(dataset_name, columns, inspector) self._set_partition_key(columns, schema_fields) @@ -754,7 +754,9 @@ def loop_views( # add view schema fields schema_fields = self.get_schema_fields( - dataset.dataset_name, dataset.columns + dataset.dataset_name, + dataset.columns, + inspector, ) schema_metadata = get_schema_metadata( @@ -877,6 +879,7 @@ def get_schema_fields_for_column( self, dataset_name: str, column: Dict[Any, Any], + inspector: Inspector, pk_constraints: Optional[Dict[Any, Any]] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index de3012cc335681..1fa308eae6b769 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -120,6 +120,9 @@ from datahub.utilities.lossy_collections import LossyList from datahub.utilities.registries.domain_registry import DomainRegistry from datahub.utilities.sqlalchemy_query_combiner import SQLAlchemyQueryCombinerReport +from datahub.utilities.sqlalchemy_type_converter import ( + get_native_data_type_for_sqlalchemy_type, +) if TYPE_CHECKING: from datahub.ingestion.source.ge_data_profiler import ( @@ -807,6 +810,7 @@ def _process_table( schema_fields = self.get_schema_fields( dataset_name, columns, + inspector, pk_constraints, tags=extra_tags, partition_keys=partitions, @@ -987,6 +991,7 @@ def get_schema_fields( self, dataset_name: str, columns: List[dict], + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[Dict[str, List[str]]] = None, @@ -999,6 +1004,7 @@ def get_schema_fields( fields = self.get_schema_fields_for_column( dataset_name, column, + inspector, pk_constraints, tags=column_tags, partition_keys=partition_keys, @@ -1010,6 +1016,7 @@ def get_schema_fields_for_column( self, dataset_name: str, column: dict, + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, @@ -1019,10 +1026,16 @@ def get_schema_fields_for_column( tags_str = [make_tag_urn(t) for t in tags] tags_tac = [TagAssociationClass(t) for t in tags_str] gtc = GlobalTagsClass(tags_tac) + full_type = column.get("full_type") field = SchemaField( fieldPath=column["name"], type=get_column_type(self.report, dataset_name, column["type"]), - nativeDataType=column.get("full_type", repr(column["type"])), + nativeDataType=full_type + if full_type is not None + else get_native_data_type_for_sqlalchemy_type( + column["type"], + inspector=inspector, + ), description=column.get("comment", None), nullable=column["nullable"], recursive=False, @@ -1099,7 +1112,7 @@ def _process_view( self.warn(logger, dataset_name, "unable to get schema for this view") schema_metadata = None else: - schema_fields = self.get_schema_fields(dataset_name, columns) + schema_fields = self.get_schema_fields(dataset_name, columns, inspector) schema_metadata = get_schema_metadata( self.report, dataset_name, diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py index cc0a43bc5e8749..b6fa51dd70e18d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/trino.py @@ -387,12 +387,16 @@ def get_schema_fields_for_column( self, dataset_name: str, column: dict, + inspector: Inspector, pk_constraints: Optional[dict] = None, partition_keys: Optional[List[str]] = None, tags: Optional[List[str]] = None, ) -> List[SchemaField]: fields = super().get_schema_fields_for_column( - dataset_name, column, pk_constraints + dataset_name, + column, + inspector, + pk_constraints, ) if isinstance(column["type"], (datatype.ROW, sqltypes.ARRAY, datatype.MAP)): diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py index ae56fb87ee5281..a340f049731c46 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/vertica.py @@ -469,7 +469,12 @@ def _process_projections( foreign_keys = self._get_foreign_keys( dataset_urn, inspector, schema, projection ) - schema_fields = self.get_schema_fields(dataset_name, columns, pk_constraints) + schema_fields = self.get_schema_fields( + dataset_name, + columns, + inspector, + pk_constraints, + ) schema_metadata = get_schema_metadata( self.report, dataset_name, @@ -673,7 +678,7 @@ def _process_models( ) dataset_snapshot.aspects.append(dataset_properties) - schema_fields = self.get_schema_fields(dataset_name, columns) + schema_fields = self.get_schema_fields(dataset_name, columns, inspector) schema_metadata = get_schema_metadata( self.report, diff --git a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py index 41d02646fdb8a0..ad94c6904e2807 100644 --- a/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py +++ b/metadata-ingestion/src/datahub/utilities/sqlalchemy_type_converter.py @@ -5,6 +5,8 @@ from typing import Any, Dict, List, Optional, Type, Union from sqlalchemy import types +from sqlalchemy.engine.reflection import Inspector +from sqlalchemy.sql.visitors import Visitable from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields from datahub.metadata.com.linkedin.pegasus2avro.schema import SchemaField @@ -176,6 +178,7 @@ def get_avro_for_sqlalchemy_column( def get_schema_fields_for_sqlalchemy_column( column_name: str, column_type: types.TypeEngine, + inspector: Inspector, description: Optional[str] = None, nullable: Optional[bool] = True, is_part_of_key: Optional[bool] = False, @@ -216,7 +219,10 @@ def get_schema_fields_for_sqlalchemy_column( SchemaField( fieldPath=column_name, type=SchemaFieldDataTypeClass(type=NullTypeClass()), - nativeDataType=str(column_type), + nativeDataType=get_native_data_type_for_sqlalchemy_type( + column_type, + inspector, + ), ) ] @@ -240,3 +246,25 @@ def get_schema_fields_for_sqlalchemy_column( ) return schema_fields + + +def get_native_data_type_for_sqlalchemy_type( + column_type: types.TypeEngine, inspector: Inspector +) -> str: + if isinstance(column_type, types.NullType): + return column_type.__visit_name__ + + try: + return column_type.compile(dialect=inspector.dialect) + except Exception as e: + logger.debug( + f"Unable to compile sqlalchemy type {column_type} the error was: {e}" + ) + + if ( + isinstance(column_type, Visitable) + and column_type.__visit_name__ is not None + ): + return column_type.__visit_name__ + + return repr(column_type) diff --git a/metadata-ingestion/tests/integration/hana/docker-compose.yml b/metadata-ingestion/tests/integration/hana/docker-compose.yml index 38bd1f544a0955..3f742362284835 100644 --- a/metadata-ingestion/tests/integration/hana/docker-compose.yml +++ b/metadata-ingestion/tests/integration/hana/docker-compose.yml @@ -2,15 +2,11 @@ version: '3.4' services: testhana: - image: "store/saplabs/hanaexpress:2.00.054.00.20210603.1" + image: "saplabs/hanaexpress:latest" container_name: "testhana" restart: "unless-stopped" ports: - - 39013:39013 - - 39017:39017 - - 39041-39045:39041-39045 - - 1128-1129:1128-1129 - - 59013-59014:59013-59014 + - 39041:39041 volumes: - ./post_start:/hana/hooks/post_start/ - ./setup:/hana/mounts/setup/ diff --git a/metadata-ingestion/tests/integration/hana/hana_mces_golden.json b/metadata-ingestion/tests/integration/hana/hana_mces_golden.json index 84ad1f3d3e592c..26789abc355c75 100644 --- a/metadata-ingestion/tests/integration/hana/hana_mces_golden.json +++ b/metadata-ingestion/tests/integration/hana/hana_mces_golden.json @@ -1,158 +1,217 @@ [ { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hana\", \"instance\": \"PROD\", \"database\": \"hxe\"}, \"name\": \"hxe\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "platform": "hana", + "env": "PROD", + "database": "hxe" + }, + "name": "hxe" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hana\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:hana" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Database" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hana\", \"instance\": \"PROD\", \"database\": \"hxe\", \"schema\": \"hotel\"}, \"name\": \"hotel\"}", - "contentType": "application/json" + "json": { + "customProperties": { + "platform": "hana", + "env": "PROD", + "database": "hxe", + "schema": "hotel" + }, + "name": "hotel" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hana\"}", - "contentType": "application/json" + "json": { + "platform": "urn:li:dataPlatform:hana" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Schema" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "container", "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:c8107a53ee221a15de176e4d34a06940\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:f870c782e0a44727bd10da2ab742363b\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", @@ -165,11 +224,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, - "externalUrl": null, "name": "customer", - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -180,17 +235,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -200,189 +250,166 @@ "fields": [ { "fieldPath": "cno", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": true }, { "fieldPath": "title", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=7)", + "nativeDataType": "VARCHAR(7)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "firstname", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=20)", + "nativeDataType": "VARCHAR(20)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "zip", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=6)", + "nativeDataType": "VARCHAR(6)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Table" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + }, + { + "id": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "urn": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:f870c782e0a44727bd10da2ab742363b\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", @@ -395,11 +422,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, - "externalUrl": null, "name": "hotel", - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -410,17 +433,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -430,189 +448,166 @@ "fields": [ { "fieldPath": "hno", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": true }, { "fieldPath": "name", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "address", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "city", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=30)", + "nativeDataType": "VARCHAR(30)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "state", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=2)", + "nativeDataType": "VARCHAR(2)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "zip", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=6)", + "nativeDataType": "VARCHAR(6)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Table" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + }, + { + "id": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "urn": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:f870c782e0a44727bd10da2ab742363b\"}", - "contentType": "application/json" + "json": { + "container": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", @@ -625,11 +620,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, - "externalUrl": null, "name": "maintenance", - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -640,17 +631,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.MySqlDDL": { @@ -660,205 +646,811 @@ "fields": [ { "fieldPath": "mno", - "jsonPath": null, "nullable": false, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": true }, { "fieldPath": "hno", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "description", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "date_performed", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false }, { "fieldPath": "performed_by", - "jsonPath": null, "nullable": true, - "description": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "isPartitioningKey": null, - "jsonProps": null + "isPartOfKey": false } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Table\"]}", - "contentType": "application/json" + "json": { + "typeNames": [ + "Table" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", "aspect": { - "value": "{\"domains\": [\"urn:li:domain:sales\"]}", - "contentType": "application/json" + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:c8107a53ee221a15de176e4d34a06940", + "urn": "urn:li:container:c8107a53ee221a15de176e4d34a06940" + }, + { + "id": "urn:li:container:f870c782e0a44727bd10da2ab742363b", + "urn": "urn:li:container:f870c782e0a44727bd10da2ab742363b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hana-test", + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.customer,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 15, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"cno\", \"uniqueCount\": 15, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"1000\", \"1001\", \"1002\", \"1003\", \"1004\", \"1005\", \"1006\", \"1007\", \"1008\", \"1009\", \"1010\", \"1011\", \"1012\", \"1013\", \"1014\"]}, {\"fieldPath\": \"title\", \"uniqueCount\": 3, \"uniqueProportion\": 0.2, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Company\", \"frequency\": 2}, {\"value\": \"Mr\", \"frequency\": 7}, {\"value\": \"Mrs\", \"frequency\": 6}], \"sampleValues\": [\"Mrs\", \"Mr\", \"Company\", \"Mrs\", \"Mrs\", \"Mr\", \"Mrs\", \"Mr\", \"Mrs\", \"Mr\", \"Mr\", \"Mrs\", \"Mr\", \"Company\", \"Mr\"]}, {\"fieldPath\": \"firstname\", \"uniqueCount\": 13, \"uniqueProportion\": 1.0, \"nullCount\": 2, \"nullProportion\": 0.13333333333333333, \"sampleValues\": [\"Jenny\", \"Peter\", \"Rose\", \"Mary\", \"Martin\", \"Sally\", \"Mike\", \"Rita\", \"George\", \"Frank\", \"Susan\", \"Joseph\", \"Antony\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 15, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Porter\", \"Brown\", \"Datasoft\", \"Brian\", \"Griffith\", \"Randolph\", \"Smith\", \"Jackson\", \"Doe\", \"Howe\", \"Miller\", \"Baker\", \"Peters\", \"TOOLware\", \"Jenkins\"]}, {\"fieldPath\": \"address\", \"uniqueCount\": 15, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"1340 N. Ash Street, #3\", \"1001 34th St., APT.3\", \"486 Maple St.\", \"500 Yellowstone Drive, #2\", \"3401 Elder Lane\", \"340 MAIN STREET, #7\", \"250 Curtis Street\", \"133 BROADWAY APT. 1\", \"2000 Humboldt St., #6\", \"111 B Parkway, #23\", \"27 5th St., 76\", \"200 MAIN STREET, #94\", \"700 S. Ash St., APT.12\", \"410 Mariposa St., #10\", \"55 A Parkway, #15\"]}, {\"fieldPath\": \"zip\", \"uniqueCount\": 12, \"uniqueProportion\": 0.8, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"10580\", \"frequency\": 1}, {\"value\": \"20005\", \"frequency\": 1}, {\"value\": \"20019\", \"frequency\": 1}, {\"value\": \"20903\", \"frequency\": 1}, {\"value\": \"45211\", \"frequency\": 1}, {\"value\": \"48226\", \"frequency\": 1}, {\"value\": \"60615\", \"frequency\": 1}, {\"value\": \"75243\", \"frequency\": 3}, {\"value\": \"90018\", \"frequency\": 2}, {\"value\": \"92714\", \"frequency\": 1}, {\"value\": \"95054\", \"frequency\": 1}, {\"value\": \"97213\", \"frequency\": 1}], \"sampleValues\": [\"10580\", \"48226\", \"90018\", \"75243\", \"20005\", \"60615\", \"75243\", \"45211\", \"97213\", \"75243\", \"95054\", \"90018\", \"92714\", \"20019\", \"20903\"]}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 15, + "columnCount": 6, + "fieldProfiles": [ + { + "fieldPath": "cno", + "uniqueCount": 15, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "1000", + "max": "1014", + "mean": "1007.0", + "median": "1007", + "stdev": "4.472135", + "sampleValues": [ + "1000", + "1001", + "1002", + "1003", + "1004", + "1005", + "1006", + "1007", + "1008", + "1009", + "1010", + "1011", + "1012", + "1013", + "1014" + ] + }, + { + "fieldPath": "title", + "uniqueCount": 3, + "uniqueProportion": 0.2, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "Company", + "frequency": 2 + }, + { + "value": "Mr", + "frequency": 7 + }, + { + "value": "Mrs", + "frequency": 6 + } + ], + "sampleValues": [ + "Mrs", + "Mr", + "Company", + "Mrs", + "Mrs", + "Mr", + "Mrs", + "Mr", + "Mrs", + "Mr", + "Mr", + "Mrs", + "Mr", + "Company", + "Mr" + ] + }, + { + "fieldPath": "firstname", + "uniqueCount": 13, + "uniqueProportion": 1, + "nullCount": 2, + "nullProportion": 0.13333333333333333, + "sampleValues": [ + "Jenny", + "Peter", + "Rose", + "Mary", + "Martin", + "Sally", + "Mike", + "Rita", + "George", + "Frank", + "Susan", + "Joseph", + "Antony" + ] + }, + { + "fieldPath": "name", + "uniqueCount": 15, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Porter", + "Brown", + "Datasoft", + "Brian", + "Griffith", + "Randolph", + "Smith", + "Jackson", + "Doe", + "Howe", + "Miller", + "Baker", + "Peters", + "TOOLware", + "Jenkins" + ] + }, + { + "fieldPath": "address", + "uniqueCount": 15, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "1340 N. Ash Street, #3", + "1001 34th St., APT.3", + "486 Maple St.", + "500 Yellowstone Drive, #2", + "3401 Elder Lane", + "340 MAIN STREET, #7", + "250 Curtis Street", + "133 BROADWAY APT. 1", + "2000 Humboldt St., #6", + "111 B Parkway, #23", + "27 5th St., 76", + "200 MAIN STREET, #94", + "700 S. Ash St., APT.12", + "410 Mariposa St., #10", + "55 A Parkway, #15" + ] + }, + { + "fieldPath": "zip", + "uniqueCount": 12, + "uniqueProportion": 0.8, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "10580", + "frequency": 1 + }, + { + "value": "20005", + "frequency": 1 + }, + { + "value": "20019", + "frequency": 1 + }, + { + "value": "20903", + "frequency": 1 + }, + { + "value": "45211", + "frequency": 1 + }, + { + "value": "48226", + "frequency": 1 + }, + { + "value": "60615", + "frequency": 1 + }, + { + "value": "75243", + "frequency": 3 + }, + { + "value": "90018", + "frequency": 2 + }, + { + "value": "92714", + "frequency": 1 + }, + { + "value": "95054", + "frequency": 1 + }, + { + "value": "97213", + "frequency": 1 + } + ], + "sampleValues": [ + "10580", + "48226", + "90018", + "75243", + "20005", + "60615", + "75243", + "45211", + "97213", + "75243", + "95054", + "90018", + "92714", + "20019", + "20903" + ] + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.hotel,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 17, \"columnCount\": 6, \"fieldProfiles\": [{\"fieldPath\": \"hno\", \"uniqueCount\": 17, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\", \"17\", \"18\", \"19\", \"20\", \"21\", \"22\", \"23\", \"24\", \"25\", \"26\"]}, {\"fieldPath\": \"name\", \"uniqueCount\": 17, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Congress\", \"Regency\", \"Long Island\", \"Empire State\", \"Midtown\", \"Eighth Avenue\", \"Lake Michigan\", \"Airport\", \"Sunshine\", \"Beach\", \"Atlantic\", \"Long Beach\", \"Indian Horse\", \"Star\", \"River Boat\", \"Ocean Star\", \"Bella Ciente\"]}, {\"fieldPath\": \"address\", \"uniqueCount\": 17, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"155 Beechwood St.\", \"477 17th Avenue\", \"1499 Grove Street\", \"65 Yellowstone Dr.\", \"12 Barnard St.\", \"112 8th Avenue\", \"354 OAK Terrace\", \"650 C Parkway\", \"200 Yellowstone Dr.\", \"1980 34th St.\", \"111 78th St.\", \"35 Broadway\", \"16 MAIN STREET\", \"13 Beechwood Place\", \"788 MAIN STREET\", \"45 Pacific Avenue\", \"1407 Marshall Ave\"]}, {\"fieldPath\": \"city\", \"uniqueCount\": 15, \"uniqueProportion\": 0.8823529411764706, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"Albany\", \"frequency\": 1}, {\"value\": \"Atlantic City\", \"frequency\": 1}, {\"value\": \"Chicago\", \"frequency\": 1}, {\"value\": \"Clearwater\", \"frequency\": 1}, {\"value\": \"Daytona Beach\", \"frequency\": 1}, {\"value\": \"Deerfield Beach\", \"frequency\": 1}, {\"value\": \"Hollywood\", \"frequency\": 1}, {\"value\": \"Long Beach\", \"frequency\": 1}, {\"value\": \"Long Island\", \"frequency\": 1}, {\"value\": \"Longview\", \"frequency\": 1}, {\"value\": \"New Orleans\", \"frequency\": 1}, {\"value\": \"New York\", \"frequency\": 2}, {\"value\": \"Palm Springs\", \"frequency\": 1}, {\"value\": \"Rosemont\", \"frequency\": 1}, {\"value\": \"Seattle\", \"frequency\": 2}], \"sampleValues\": [\"Seattle\", \"Seattle\", \"Long Island\", \"Albany\", \"New York\", \"New York\", \"Chicago\", \"Rosemont\", \"Clearwater\", \"Daytona Beach\", \"Deerfield Beach\", \"Long Beach\", \"Palm Springs\", \"Hollywood\", \"New Orleans\", \"Atlantic City\", \"Longview\"]}, {\"fieldPath\": \"state\", \"uniqueCount\": 8, \"uniqueProportion\": 0.47058823529411764, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"CA\", \"frequency\": 3}, {\"value\": \"FL\", \"frequency\": 3}, {\"value\": \"IL\", \"frequency\": 2}, {\"value\": \"LA\", \"frequency\": 1}, {\"value\": \"NJ\", \"frequency\": 1}, {\"value\": \"NY\", \"frequency\": 4}, {\"value\": \"TX\", \"frequency\": 1}, {\"value\": \"WA\", \"frequency\": 2}], \"sampleValues\": [\"WA\", \"WA\", \"NY\", \"NY\", \"NY\", \"NY\", \"IL\", \"IL\", \"FL\", \"FL\", \"FL\", \"CA\", \"CA\", \"CA\", \"LA\", \"NJ\", \"TX\"]}, {\"fieldPath\": \"zip\", \"uniqueCount\": 16, \"uniqueProportion\": 0.9411764705882353, \"nullCount\": 0, \"nullProportion\": 0.0, \"distinctValueFrequencies\": [{\"value\": \"08401\", \"frequency\": 1}, {\"value\": \"10019\", \"frequency\": 2}, {\"value\": \"11788\", \"frequency\": 1}, {\"value\": \"12203\", \"frequency\": 1}, {\"value\": \"20005\", \"frequency\": 1}, {\"value\": \"20037\", \"frequency\": 1}, {\"value\": \"32018\", \"frequency\": 1}, {\"value\": \"33441\", \"frequency\": 1}, {\"value\": \"33575\", \"frequency\": 1}, {\"value\": \"60018\", \"frequency\": 1}, {\"value\": \"60601\", \"frequency\": 1}, {\"value\": \"70112\", \"frequency\": 1}, {\"value\": \"75601\", \"frequency\": 1}, {\"value\": \"90029\", \"frequency\": 1}, {\"value\": \"90804\", \"frequency\": 1}, {\"value\": \"92262\", \"frequency\": 1}], \"sampleValues\": [\"20005\", \"20037\", \"11788\", \"12203\", \"10019\", \"10019\", \"60601\", \"60018\", \"33575\", \"32018\", \"33441\", \"90804\", \"92262\", \"90029\", \"70112\", \"08401\", \"75601\"]}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 17, + "columnCount": 6, + "fieldProfiles": [ + { + "fieldPath": "hno", + "uniqueCount": 17, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "10", + "max": "26", + "mean": "18.0", + "median": "18", + "stdev": "5.049752", + "sampleValues": [ + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + "24", + "25", + "26" + ] + }, + { + "fieldPath": "name", + "uniqueCount": 17, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Congress", + "Regency", + "Long Island", + "Empire State", + "Midtown", + "Eighth Avenue", + "Lake Michigan", + "Airport", + "Sunshine", + "Beach", + "Atlantic", + "Long Beach", + "Indian Horse", + "Star", + "River Boat", + "Ocean Star", + "Bella Ciente" + ] + }, + { + "fieldPath": "address", + "uniqueCount": 17, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "155 Beechwood St.", + "477 17th Avenue", + "1499 Grove Street", + "65 Yellowstone Dr.", + "12 Barnard St.", + "112 8th Avenue", + "354 OAK Terrace", + "650 C Parkway", + "200 Yellowstone Dr.", + "1980 34th St.", + "111 78th St.", + "35 Broadway", + "16 MAIN STREET", + "13 Beechwood Place", + "788 MAIN STREET", + "45 Pacific Avenue", + "1407 Marshall Ave" + ] + }, + { + "fieldPath": "city", + "uniqueCount": 15, + "uniqueProportion": 0.8823529411764706, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "Albany", + "frequency": 1 + }, + { + "value": "Atlantic City", + "frequency": 1 + }, + { + "value": "Chicago", + "frequency": 1 + }, + { + "value": "Clearwater", + "frequency": 1 + }, + { + "value": "Daytona Beach", + "frequency": 1 + }, + { + "value": "Deerfield Beach", + "frequency": 1 + }, + { + "value": "Hollywood", + "frequency": 1 + }, + { + "value": "Long Beach", + "frequency": 1 + }, + { + "value": "Long Island", + "frequency": 1 + }, + { + "value": "Longview", + "frequency": 1 + }, + { + "value": "New Orleans", + "frequency": 1 + }, + { + "value": "New York", + "frequency": 2 + }, + { + "value": "Palm Springs", + "frequency": 1 + }, + { + "value": "Rosemont", + "frequency": 1 + }, + { + "value": "Seattle", + "frequency": 2 + } + ], + "sampleValues": [ + "Seattle", + "Seattle", + "Long Island", + "Albany", + "New York", + "New York", + "Chicago", + "Rosemont", + "Clearwater", + "Daytona Beach", + "Deerfield Beach", + "Long Beach", + "Palm Springs", + "Hollywood", + "New Orleans", + "Atlantic City", + "Longview" + ] + }, + { + "fieldPath": "state", + "uniqueCount": 8, + "uniqueProportion": 0.47058823529411764, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "CA", + "frequency": 3 + }, + { + "value": "FL", + "frequency": 3 + }, + { + "value": "IL", + "frequency": 2 + }, + { + "value": "LA", + "frequency": 1 + }, + { + "value": "NJ", + "frequency": 1 + }, + { + "value": "NY", + "frequency": 4 + }, + { + "value": "TX", + "frequency": 1 + }, + { + "value": "WA", + "frequency": 2 + } + ], + "sampleValues": [ + "WA", + "WA", + "NY", + "NY", + "NY", + "NY", + "IL", + "IL", + "FL", + "FL", + "FL", + "CA", + "CA", + "CA", + "LA", + "NJ", + "TX" + ] + }, + { + "fieldPath": "zip", + "uniqueCount": 16, + "uniqueProportion": 0.9411764705882353, + "nullCount": 0, + "nullProportion": 0.0, + "distinctValueFrequencies": [ + { + "value": "08401", + "frequency": 1 + }, + { + "value": "10019", + "frequency": 2 + }, + { + "value": "11788", + "frequency": 1 + }, + { + "value": "12203", + "frequency": 1 + }, + { + "value": "20005", + "frequency": 1 + }, + { + "value": "20037", + "frequency": 1 + }, + { + "value": "32018", + "frequency": 1 + }, + { + "value": "33441", + "frequency": 1 + }, + { + "value": "33575", + "frequency": 1 + }, + { + "value": "60018", + "frequency": 1 + }, + { + "value": "60601", + "frequency": 1 + }, + { + "value": "70112", + "frequency": 1 + }, + { + "value": "75601", + "frequency": 1 + }, + { + "value": "90029", + "frequency": 1 + }, + { + "value": "90804", + "frequency": 1 + }, + { + "value": "92262", + "frequency": 1 + } + ], + "sampleValues": [ + "20005", + "20037", + "11788", + "12203", + "10019", + "10019", + "60601", + "60018", + "33575", + "32018", + "33441", + "90804", + "92262", + "90029", + "70112", + "08401", + "75601" + ] + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hana,hotel.maintenance,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { - "value": "{\"timestampMillis\": 1586847600000, \"partitionSpec\": {\"type\": \"FULL_TABLE\", \"partition\": \"FULL_TABLE_SNAPSHOT\"}, \"rowCount\": 3, \"columnCount\": 5, \"fieldProfiles\": [{\"fieldPath\": \"mno\", \"uniqueCount\": 3, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"10\", \"11\", \"12\"]}, {\"fieldPath\": \"hno\", \"uniqueCount\": 3, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"24\", \"25\", \"26\"]}, {\"fieldPath\": \"description\", \"uniqueCount\": 3, \"uniqueProportion\": 1.0, \"nullCount\": 0, \"nullProportion\": 0.0, \"sampleValues\": [\"Replace pool liner and pump\", \"Renovate the bar area. Replace TV and speakers\", \"Roof repair due to storm\"]}, {\"fieldPath\": \"date_performed\", \"uniqueCount\": 2, \"uniqueProportion\": 1.0, \"nullCount\": 1, \"nullProportion\": 0.3333333333333333, \"min\": \"2019-03-21\", \"max\": \"2020-11-29\", \"sampleValues\": [\"2019-03-21\", \"2020-11-29\"]}, {\"fieldPath\": \"performed_by\", \"uniqueCount\": 2, \"uniqueProportion\": 1.0, \"nullCount\": 1, \"nullProportion\": 0.3333333333333333, \"sampleValues\": [\"Discount Pool Supplies\", \"TV and Audio Superstore\"]}]}", - "contentType": "application/json" + "json": { + "timestampMillis": 1586847600000, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "rowCount": 3, + "columnCount": 5, + "fieldProfiles": [ + { + "fieldPath": "mno", + "uniqueCount": 3, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "10", + "max": "12", + "mean": "11.0", + "median": "11", + "stdev": "1.0", + "sampleValues": [ + "10", + "11", + "12" + ] + }, + { + "fieldPath": "hno", + "uniqueCount": 3, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "min": "24", + "max": "26", + "mean": "25.0", + "median": "25", + "stdev": "1.0", + "sampleValues": [ + "24", + "25", + "26" + ] + }, + { + "fieldPath": "description", + "uniqueCount": 3, + "uniqueProportion": 1, + "nullCount": 0, + "nullProportion": 0.0, + "sampleValues": [ + "Replace pool liner and pump", + "Renovate the bar area. Replace TV and speakers", + "Roof repair due to storm" + ] + }, + { + "fieldPath": "date_performed", + "uniqueCount": 2, + "uniqueProportion": 1, + "nullCount": 1, + "nullProportion": 0.3333333333333333, + "min": "2019-03-21", + "max": "2020-11-29", + "sampleValues": [ + "2019-03-21", + "2020-11-29" + ] + }, + { + "fieldPath": "performed_by", + "uniqueCount": 2, + "uniqueProportion": 1, + "nullCount": 1, + "nullProportion": 0.3333333333333333, + "sampleValues": [ + "Discount Pool Supplies", + "TV and Audio Superstore" + ] + } + ] + } }, "systemMetadata": { "lastObserved": 1586847600000, "runId": "hana-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json index f78b45fb373fc6..bddccc856c842c 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json @@ -147,7 +147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -159,7 +159,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -171,7 +171,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=14)", + "nativeDataType": "VARCHAR(14)", "recursive": false, "isPartOfKey": false }, @@ -183,7 +183,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -195,7 +195,7 @@ "com.linkedin.pegasus2avro.schema.EnumType": {} } }, - "nativeDataType": "ENUM('M', 'F')", + "nativeDataType": "ENUM('M','F')", "recursive": false, "glossaryTerms": { "terms": [ @@ -218,7 +218,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -334,7 +334,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -346,7 +346,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -358,7 +358,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": true }, @@ -370,7 +370,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -439,7 +439,6 @@ }, "rowCount": 10, "columnCount": 6, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "emp_no", @@ -574,7 +573,8 @@ "1989-08-24" ] } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -597,7 +597,6 @@ }, "rowCount": 112, "columnCount": 4, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "emp_no", @@ -991,7 +990,8 @@ "1993-02-09" ] } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -1166,7 +1166,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": true }, @@ -1178,7 +1178,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": true }, @@ -1190,7 +1190,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -1202,7 +1202,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "LONGTEXT()", + "nativeDataType": "LONGTEXT", "recursive": false, "isPartOfKey": false }, @@ -1214,7 +1214,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "DATETIME(fsp=6)", + "nativeDataType": "DATETIME(6)", "recursive": false, "isPartOfKey": false }, @@ -1226,7 +1226,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -1238,7 +1238,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false } @@ -1373,7 +1373,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -1386,7 +1386,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false }, @@ -1398,7 +1398,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=150)", + "nativeDataType": "VARCHAR(150)", "recursive": false, "isPartOfKey": false }, @@ -1410,7 +1410,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=150)", + "nativeDataType": "VARCHAR(150)", "recursive": false, "isPartOfKey": false }, @@ -1422,7 +1422,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": false }, @@ -1434,7 +1434,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false }, @@ -1446,7 +1446,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DOUBLE(asdecimal=True)", + "nativeDataType": "DOUBLE", "recursive": false, "isPartOfKey": false } @@ -1583,7 +1583,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": false }, @@ -1595,7 +1595,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false }, @@ -1607,7 +1607,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=150)", + "nativeDataType": "VARCHAR(150)", "recursive": false, "isPartOfKey": false }, @@ -1619,7 +1619,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "DOUBLE(asdecimal=True)", + "nativeDataType": "DOUBLE", "recursive": false, "isPartOfKey": false } @@ -1858,7 +1858,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1870,7 +1870,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -1882,7 +1882,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -1894,7 +1894,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -1906,7 +1906,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "glossaryTerms": { "terms": [ @@ -1929,7 +1929,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -2045,7 +2045,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2057,7 +2057,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -2069,7 +2069,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -2150,7 +2150,6 @@ }, "rowCount": 5, "columnCount": 6, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "id", @@ -2259,7 +2258,8 @@ "3.8" ] } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -2282,7 +2282,6 @@ }, "rowCount": 0, "columnCount": 3, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "id", @@ -2299,7 +2298,8 @@ "uniqueCount": 0, "nullCount": 0 } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -2456,7 +2456,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "SET('a', 'b', 'c', 'd')", + "nativeDataType": "SET('a','b','c','d')", "recursive": false, "isPartOfKey": false } @@ -2572,7 +2572,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false } @@ -2641,14 +2641,14 @@ }, "rowCount": 0, "columnCount": 1, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "col", "uniqueCount": 0, "nullCount": 0 } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { @@ -2671,14 +2671,14 @@ }, "rowCount": 0, "columnCount": 1, - "sizeInBytes": 16384, "fieldProfiles": [ { "fieldPath": "dummy", "uniqueCount": 0, "nullCount": 0 } - ] + ], + "sizeInBytes": 16384 } }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json index 065d6cbe90b313..8c6f6338bc2b07 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_with_db_golden.json @@ -165,7 +165,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -177,7 +177,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -189,7 +189,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -201,7 +201,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -213,7 +213,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -225,7 +225,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -359,7 +359,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -371,7 +371,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -383,7 +383,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json index b8dfd7d9efc37d..3f5899aa8a98c3 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_level_only.json @@ -147,7 +147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -159,7 +159,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -171,7 +171,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -183,7 +183,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -195,7 +195,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -207,7 +207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -323,7 +323,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -335,7 +335,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -347,7 +347,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -461,63 +461,5 @@ "runId": "mysql-2020_04_14-07_00_00", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", - "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f", - "urn": "urn:li:container:dc2ae101b66746b9c2b6df8ee89ca88f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-2020_04_14-07_00_00", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json index fc25af0400bb5d..58a70cae2b2d5e 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_table_row_count_estimate_only.json @@ -147,7 +147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -159,7 +159,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -171,7 +171,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -183,7 +183,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -195,7 +195,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -207,7 +207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -323,7 +323,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -335,7 +335,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -347,7 +347,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json index b740dfe025ef7f..6732b17d2e8322 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_database.json @@ -258,7 +258,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -380,7 +380,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -505,7 +505,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -754,7 +754,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -876,7 +876,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -1001,7 +1001,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -1207,213 +1207,5 @@ "runId": "oracle-2022_02_03-07_00_00", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.test2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema1.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.test4,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,OraDoc.schema2.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json index 008cd405186c39..7610daaa54b4a1 100644 --- a/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json +++ b/metadata-ingestion/tests/integration/oracle/golden_test_ingest_with_out_database.json @@ -258,7 +258,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -380,7 +380,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -505,7 +505,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -754,7 +754,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -876,7 +876,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": true } @@ -1001,7 +1001,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "NUMBER(asdecimal=False)", + "nativeDataType": "NUMBER", "recursive": false, "isPartOfKey": false } @@ -1053,214 +1053,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.test2,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema1.view1,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:c093e810646c7ebc493237bb24a3538f", - "urn": "urn:li:container:c093e810646c7ebc493237bb24a3538f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test3,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.test4,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:8c867b02fcc2615b19cd02b15b023287", - "urn": "urn:li:container:8c867b02fcc2615b19cd02b15b023287" - }, - { - "id": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825", - "urn": "urn:li:container:ab1a240f35ae787df0eff0e6726a9825" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "oracle-2022_02_03-07_00_00", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:oracle,schema2.view1,PROD)", diff --git a/metadata-ingestion/tests/integration/oracle/test_oracle.py b/metadata-ingestion/tests/integration/oracle/test_oracle.py index 6c9aba8ec5620e..4541bb8ac65bff 100644 --- a/metadata-ingestion/tests/integration/oracle/test_oracle.py +++ b/metadata-ingestion/tests/integration/oracle/test_oracle.py @@ -24,6 +24,7 @@ def apply_mock_data(self, mock_create_engine, mock_inspect, mock_event): inspector_magic_mock.dialect.server_version_info = ( self.get_server_version_info() ) + inspector_magic_mock.dialect.type_compiler.process = lambda x: "NUMBER" mock_inspect.return_value = inspector_magic_mock mock_create_engine.connect.return_value = connection_magic_mock diff --git a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json index f35ff9fdb9d153..f42ff7c0df068c 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_all_db_mces_with_db_golden.json @@ -452,7 +452,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": true }, @@ -464,7 +464,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": true }, @@ -476,7 +476,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -488,7 +488,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -500,7 +500,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -512,7 +512,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP()", + "nativeDataType": "TIMESTAMP WITHOUT TIME ZONE", "recursive": false, "isPartOfKey": false }, @@ -524,7 +524,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -536,7 +536,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -548,7 +548,7 @@ "com.linkedin.pegasus2avro.schema.RecordType": {} } }, - "nativeDataType": "JSON(astext_type=Text())", + "nativeDataType": "JSON", "recursive": false, "isPartOfKey": false } @@ -671,7 +671,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": false }, @@ -683,7 +683,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json index f47789fc470cd8..f107fb1006bf6e 100644 --- a/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/postgres/postgres_mces_with_db_golden.json @@ -256,7 +256,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "glossaryTerms": { "terms": [ @@ -279,7 +279,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": true }, @@ -291,7 +291,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "BIGINT()", + "nativeDataType": "BIGINT", "recursive": false, "isPartOfKey": true }, @@ -303,7 +303,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -315,7 +315,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "TEXT()", + "nativeDataType": "TEXT", "recursive": false, "isPartOfKey": false }, @@ -327,7 +327,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP()", + "nativeDataType": "TIMESTAMP WITHOUT TIME ZONE", "recursive": false, "isPartOfKey": false }, @@ -339,7 +339,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "glossaryTerms": { "terms": [ @@ -362,7 +362,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255)", + "nativeDataType": "VARCHAR(255)", "recursive": false, "isPartOfKey": false }, @@ -374,7 +374,7 @@ "com.linkedin.pegasus2avro.schema.RecordType": {} } }, - "nativeDataType": "JSON(astext_type=Text())", + "nativeDataType": "JSON", "recursive": false, "isPartOfKey": false } @@ -497,7 +497,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=500)", + "nativeDataType": "VARCHAR(500)", "recursive": false, "isPartOfKey": false }, @@ -509,7 +509,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=200)", + "nativeDataType": "VARCHAR(200)", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json index 4c0c1c6512ec77..74cb216117bd43 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_to_file.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -3560,7 +3560,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3572,7 +3572,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3584,7 +3584,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "MONEY()", + "nativeDataType": "MONEY", "recursive": false, "isPartOfKey": false } @@ -3813,7 +3813,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3825,7 +3825,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3837,7 +3837,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "SMALLMONEY()", + "nativeDataType": "SMALLMONEY", "recursive": false, "isPartOfKey": false } @@ -3957,7 +3957,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -3969,7 +3969,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3981,7 +3981,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -3993,7 +3993,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -4442,800 +4442,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:47217386c89d8b94153f6ee31e7e77ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:5eb0d61efa998d1ccd5cbdc6ce4bb4af", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2816b2cb7f90d3dce64125ba89fb1fa8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:20d0f0c94e9796ff44ff32d4d0e19084", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3600d2ebb33b25dac607624d7eae7575", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:280f2e3aefacc346d0ce1590ec337c7d", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:cba5c3ca7f028fcf749593be369d3c24", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:58c30fa72f213ca7e12fb04f5a7d150f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9387ddfeb7b57672cabd761ade89c49c", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.dbo.ProductsNew,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - }, - { - "id": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0", - "urn": "urn:li:container:3a5f70e0e34834d4eeeb4d5a5caf03d0" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.ItemsNew,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - }, - { - "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", - "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,NewData.FooNew.PersonsNew,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - }, - { - "id": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63", - "urn": "urn:li:container:7cc43e5b4e2a7f2f66f1df774d1a0c63" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:54727d9fd7deacef27641559125bbc56", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:141b0980dcb08f48544583e47cf48807", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:c6627af82d44de89492e1a9315ae9f4b", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59", - "urn": "urn:li:container:9447d283fb4f95ce7474f1db0179bb59" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json index 02c357259c3f53..e1af3f72a8af1f 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_no_db_with_filter.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -2385,415 +2385,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json index 02c357259c3f53..e1af3f72a8af1f 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_to_file.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -2385,415 +2385,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.dbo.Products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.Persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,DemoData.Foo.SalesReason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json index ad15c654e44c96..5b936c0d9f2446 100644 --- a/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json +++ b/metadata-ingestion/tests/integration/sql_server/golden_files/golden_mces_mssql_with_lower_case_urn.json @@ -112,11 +112,11 @@ "aspect": { "json": { "customProperties": { - "job_id": "0565425f-2083-45d3-bb61-76e0ee5e1117", + "job_id": "c6fb6778-14f1-4516-bb41-e5eaa97a687b", "job_name": "Weekly Demo Data Backup", "description": "No description available.", - "date_created": "2024-01-19 11:45:06.667000", - "date_modified": "2024-01-19 11:45:06.840000", + "date_created": "2024-07-27 23:58:29.780000", + "date_modified": "2024-07-27 23:58:29.943000", "step_id": "1", "step_name": "Set database to read only", "subsystem": "TSQL", @@ -1304,7 +1304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1316,7 +1316,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1546,7 +1546,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1558,7 +1558,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR()", + "nativeDataType": "NVARCHAR(max) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1678,7 +1678,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1691,7 +1691,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1703,7 +1703,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "nativeDataType": "VARCHAR(255) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false }, @@ -1715,7 +1715,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1835,7 +1835,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1847,7 +1847,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "UNIQUEIDENTIFIER()", + "nativeDataType": "UNIQUEIDENTIFIER", "recursive": false, "isPartOfKey": false }, @@ -1859,7 +1859,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "NVARCHAR(length=50)", + "nativeDataType": "NVARCHAR(50) COLLATE SQL_Latin1_General_CP1_CI_AS", "recursive": false, "isPartOfKey": false } @@ -1961,8 +1961,8 @@ "code": "CREATE PROCEDURE [Foo].[Proc.With.SpecialChar] @ID INT\nAS\n SELECT @ID AS ThatDB;\n", "input parameters": "['@ID']", "parameter @ID": "{'type': 'int'}", - "date_created": "2024-01-19 11:45:06.590000", - "date_modified": "2024-01-19 11:45:06.590000" + "date_created": "2024-07-27 23:58:29.703000", + "date_modified": "2024-07-27 23:58:29.703000" }, "externalUrl": "", "name": "demodata.Foo.Proc.With.SpecialChar", @@ -2385,415 +2385,5 @@ "runId": "mssql-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "urn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - }, - { - "id": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "urn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "urn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json index c5664b9373e8c5..d19b83f4d4e2f8 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json @@ -244,7 +244,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1710150034" + "transient_lastddltime": "1722106707" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -280,7 +280,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -505,7 +505,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1710150038" + "transient_lastddltime": "1722106711" }, "name": "classification_test", "tags": [] @@ -539,7 +539,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -551,7 +551,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -563,7 +563,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -575,7 +575,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -587,7 +587,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -764,7 +764,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "map_test", "tags": [] @@ -798,7 +798,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -991,7 +991,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "nested_struct_test", "tags": [] @@ -1025,7 +1025,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1262,7 +1262,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1710150028" + "transient_lastddltime": "1722106702" }, "name": "pokes", "tags": [] @@ -1296,7 +1296,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1308,7 +1308,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -1320,7 +1320,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -1497,7 +1497,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150031" + "transient_lastddltime": "1722106704" }, "name": "struct_test", "tags": [] @@ -1531,7 +1531,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1748,7 +1748,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "struct_test_view_materialized", "tags": [] @@ -1782,7 +1782,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2002,7 +2002,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150031" + "transient_lastddltime": "1722106704" }, "name": "_test_table_underscore", "tags": [] @@ -2036,7 +2036,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2048,7 +2048,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -2225,7 +2225,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1710150036" + "transient_lastddltime": "1722106709" }, "name": "union_test", "tags": [] @@ -2527,7 +2527,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1710150036", + "transient_lastddltime": "1722106709", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, @@ -2563,7 +2563,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index 2764433808cbdb..f72610fba7c547 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -231,7 +231,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1713211020" + "transient_lastddltime": "1722106707" }, "name": "array_struct_test", "description": "This table has array of structs", @@ -267,7 +267,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -471,7 +471,7 @@ "numrows": "3", "rawdatasize": "94", "totalsize": "97", - "transient_lastddltime": "1713211025" + "transient_lastddltime": "1722106711" }, "name": "classification_test", "tags": [] @@ -505,7 +505,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -517,7 +517,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "glossaryTerms": { "terms": [ @@ -540,7 +540,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "glossaryTerms": { "terms": [ @@ -563,7 +563,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "glossaryTerms": { "terms": [ @@ -586,7 +586,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "glossaryTerms": { "terms": [ @@ -753,7 +753,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "map_test", "tags": [] @@ -787,7 +787,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -959,7 +959,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "nested_struct_test", "tags": [] @@ -993,7 +993,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1209,7 +1209,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1713211015" + "transient_lastddltime": "1722106702" }, "name": "pokes", "tags": [] @@ -1243,7 +1243,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1255,7 +1255,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false }, @@ -1267,7 +1267,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -1423,7 +1423,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211017" + "transient_lastddltime": "1722106704" }, "name": "struct_test", "tags": [] @@ -1457,7 +1457,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1653,7 +1653,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "struct_test_view_materialized", "tags": [] @@ -1687,7 +1687,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1886,7 +1886,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211017" + "transient_lastddltime": "1722106704" }, "name": "_test_table_underscore", "tags": [] @@ -1920,7 +1920,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1932,7 +1932,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR()", + "nativeDataType": "VARCHAR", "recursive": false, "isPartOfKey": false } @@ -2088,7 +2088,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1713211023" + "transient_lastddltime": "1722106709" }, "name": "union_test", "tags": [] @@ -2369,7 +2369,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1713211023", + "transient_lastddltime": "1722106709", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, @@ -2405,7 +2405,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, diff --git a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json index 1f03f02fa9408f..b2afad81b12fad 100644 --- a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json @@ -256,7 +256,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -268,7 +268,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -280,7 +280,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -292,7 +292,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -304,7 +304,7 @@ "com.linkedin.pegasus2avro.schema.RecordType": {} } }, - "nativeDataType": "JSON()", + "nativeDataType": "JSON", "recursive": false, "isPartOfKey": false }, @@ -504,7 +504,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -516,7 +516,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -528,7 +528,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -540,7 +540,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -723,7 +723,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -735,7 +735,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false } @@ -918,7 +918,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -930,7 +930,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -942,7 +942,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -954,7 +954,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -966,7 +966,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -978,7 +978,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -1479,142 +1479,5 @@ "runId": "trino-test", "lastRunId": "no-run-id-provided" } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.book,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.issue_history,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.member,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:trino,postgresqldb.librarydb.book_in_circulation,PROD)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761", - "urn": "urn:li:container:ad9f7c5e0d4bf83d6278f62271c28761" - }, - { - "id": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c", - "urn": "urn:li:container:2d206e03e435f48a5b8bacf444bf565c" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "lastRunId": "no-run-id-provided" - } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json index cd1cd0d7e28a48..b7e0f268aa7fe1 100644 --- a/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json +++ b/metadata-ingestion/tests/integration/vertica/vertica_mces_with_db_golden.json @@ -250,7 +250,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.057395+00:00", + "create_time": "2024-06-18 12:46:11.762979+00:00", "table_size": "0 KB" }, "name": "clicks", @@ -287,7 +287,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -300,7 +300,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -313,7 +313,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false } @@ -427,7 +427,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.887434+00:00", + "create_time": "2024-06-18 12:45:49.748139+00:00", "table_size": "2119 KB" }, "name": "customer_dimension", @@ -464,7 +464,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -477,7 +477,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -490,7 +490,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -503,7 +503,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -516,7 +516,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -529,7 +529,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -542,7 +542,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -555,7 +555,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -568,7 +568,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -581,7 +581,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -594,7 +594,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -607,7 +607,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -620,7 +620,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -633,7 +633,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -646,7 +646,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -659,7 +659,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -672,7 +672,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -685,7 +685,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -698,7 +698,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -711,7 +711,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -724,7 +724,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -838,7 +838,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.897002+00:00", + "create_time": "2024-06-18 12:45:49.756539+00:00", "table_size": "145 KB" }, "name": "date_dimension", @@ -875,7 +875,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -888,7 +888,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -901,7 +901,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=18)", + "nativeDataType": "VARCHAR(18)", "recursive": false, "isPartOfKey": false }, @@ -914,7 +914,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -927,7 +927,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -940,7 +940,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -953,7 +953,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -966,7 +966,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -979,7 +979,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -992,7 +992,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1005,7 +1005,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1018,7 +1018,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -1031,7 +1031,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1044,7 +1044,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -1057,7 +1057,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1070,7 +1070,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -1083,7 +1083,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1096,7 +1096,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1109,7 +1109,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=10)", + "nativeDataType": "VARCHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -1122,7 +1122,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -1135,7 +1135,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -1249,7 +1249,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.903227+00:00", + "create_time": "2024-06-18 12:45:49.761468+00:00", "table_size": "327 KB" }, "name": "employee_dimension", @@ -1286,7 +1286,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -1299,7 +1299,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -1312,7 +1312,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -1325,7 +1325,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1338,7 +1338,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -1351,7 +1351,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1364,7 +1364,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1377,7 +1377,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -1390,7 +1390,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -1403,7 +1403,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1416,7 +1416,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -1429,7 +1429,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -1442,7 +1442,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -1455,7 +1455,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1468,7 +1468,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1481,7 +1481,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1494,7 +1494,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -1507,7 +1507,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -1621,7 +1621,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.912348+00:00", + "create_time": "2024-06-18 12:45:49.768272+00:00", "table_size": "2567 KB" }, "name": "inventory_fact", @@ -1658,7 +1658,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1671,7 +1671,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1684,7 +1684,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1697,7 +1697,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1710,7 +1710,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -1723,7 +1723,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -1837,7 +1837,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.194163+00:00", + "create_time": "2024-06-18 12:46:11.881911+00:00", "table_size": "0 KB" }, "name": "phrases", @@ -1874,7 +1874,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false } @@ -1988,7 +1988,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.890782+00:00", + "create_time": "2024-06-18 12:45:49.751522+00:00", "table_size": "19 KB" }, "name": "product_dimension", @@ -2025,7 +2025,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2038,7 +2038,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2051,7 +2051,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -2064,7 +2064,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2077,7 +2077,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2090,7 +2090,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2103,7 +2103,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2116,7 +2116,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2129,7 +2129,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2142,7 +2142,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2155,7 +2155,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2168,7 +2168,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2181,7 +2181,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2194,7 +2194,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2207,7 +2207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2220,7 +2220,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2233,7 +2233,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2246,7 +2246,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2259,7 +2259,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2272,7 +2272,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2285,7 +2285,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -2399,7 +2399,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.893891+00:00", + "create_time": "2024-06-18 12:45:49.754039+00:00", "table_size": "3 KB" }, "name": "promotion_dimension", @@ -2436,7 +2436,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2449,7 +2449,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -2462,7 +2462,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2475,7 +2475,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2488,7 +2488,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2501,7 +2501,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2514,7 +2514,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2527,7 +2527,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -2540,7 +2540,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -2553,7 +2553,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2566,7 +2566,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -2579,7 +2579,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -2693,7 +2693,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.046829+00:00", + "create_time": "2024-06-18 12:46:11.751917+00:00", "table_size": "0 KB" }, "name": "readings", @@ -2730,7 +2730,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -2743,7 +2743,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false }, @@ -2756,7 +2756,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -2870,7 +2870,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.906471+00:00", + "create_time": "2024-06-18 12:45:49.763788+00:00", "table_size": "1 KB" }, "name": "shipping_dimension", @@ -2907,7 +2907,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -2920,7 +2920,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=30)", + "nativeDataType": "CHAR(30)", "recursive": false, "isPartOfKey": false }, @@ -2933,7 +2933,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=10)", + "nativeDataType": "CHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -2946,7 +2946,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false } @@ -3060,7 +3060,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.900185+00:00", + "create_time": "2024-06-18 12:45:49.759103+00:00", "table_size": "1 KB" }, "name": "vendor_dimension", @@ -3097,7 +3097,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -3110,7 +3110,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -3123,7 +3123,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -3136,7 +3136,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -3149,7 +3149,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -3162,7 +3162,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -3175,7 +3175,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3188,7 +3188,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -3302,7 +3302,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:30.233405+00:00", + "create_time": "2024-06-18 12:46:11.212820+00:00", "table_size": "0 KB" }, "name": "vmart_load_success", @@ -3339,7 +3339,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -3453,7 +3453,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.909432+00:00", + "create_time": "2024-06-18 12:45:49.766036+00:00", "table_size": "2 KB" }, "name": "warehouse_dimension", @@ -3490,7 +3490,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -3503,7 +3503,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=20)", + "nativeDataType": "VARCHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -3516,7 +3516,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -3529,7 +3529,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=60)", + "nativeDataType": "VARCHAR(60)", "recursive": false, "isPartOfKey": false }, @@ -3542,7 +3542,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -3555,7 +3555,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -3669,7 +3669,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:24:31.075640+00:00", + "create_time": "2024-06-18 12:46:11.778456+00:00", "table_size": "0 KB", "view_definition": "SELECT sum(customer_dimension.annual_income) AS SUM, customer_dimension.customer_state FROM public.customer_dimension WHERE (customer_dimension.customer_key IN (SELECT store_sales_fact.customer_key FROM store.store_sales_fact)) GROUP BY customer_dimension.customer_state ORDER BY customer_dimension.customer_state", "is_view": "True" @@ -3708,7 +3708,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -3721,7 +3721,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false } @@ -3962,7 +3962,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.918904+00:00", + "create_time": "2024-06-18 12:45:49.773986+00:00", "table_size": "2 KB" }, "name": "store_dimension", @@ -3999,7 +3999,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -4012,7 +4012,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -4025,7 +4025,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4038,7 +4038,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -4051,7 +4051,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -4064,7 +4064,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -4077,7 +4077,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -4090,7 +4090,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4103,7 +4103,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4116,7 +4116,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4129,7 +4129,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4142,7 +4142,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4155,7 +4155,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4168,7 +4168,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4181,7 +4181,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4194,7 +4194,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4207,7 +4207,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4220,7 +4220,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -4334,7 +4334,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.929154+00:00", + "create_time": "2024-06-18 12:45:49.781744+00:00", "table_size": "8646 KB" }, "name": "store_orders_fact", @@ -4371,7 +4371,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4384,7 +4384,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4397,7 +4397,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4410,7 +4410,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4423,7 +4423,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4436,7 +4436,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4449,7 +4449,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4462,7 +4462,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4475,7 +4475,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4488,7 +4488,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4501,7 +4501,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4514,7 +4514,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4527,7 +4527,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -4540,7 +4540,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4553,7 +4553,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4566,7 +4566,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4579,7 +4579,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4592,7 +4592,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4605,7 +4605,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -4719,7 +4719,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.922050+00:00", + "create_time": "2024-06-18 12:45:49.776427+00:00", "table_size": "225096 KB" }, "name": "store_sales_fact", @@ -4756,7 +4756,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4769,7 +4769,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4782,7 +4782,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4795,7 +4795,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4808,7 +4808,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4821,7 +4821,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4834,7 +4834,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4847,7 +4847,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4860,7 +4860,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4873,7 +4873,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4886,7 +4886,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4899,7 +4899,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -4912,7 +4912,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -4925,7 +4925,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIME()", + "nativeDataType": "TIME", "recursive": false, "isPartOfKey": false }, @@ -4938,7 +4938,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -4951,7 +4951,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -4964,7 +4964,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false } @@ -5187,7 +5187,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.938730+00:00", + "create_time": "2024-06-18 12:45:49.789350+00:00", "table_size": "6 KB" }, "name": "call_center_dimension", @@ -5224,7 +5224,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -5237,7 +5237,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5250,7 +5250,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5263,7 +5263,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -5276,7 +5276,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -5289,7 +5289,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5302,7 +5302,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -5315,7 +5315,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, "isPartOfKey": false }, @@ -5328,7 +5328,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -5341,7 +5341,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -5354,7 +5354,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -5367,7 +5367,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false } @@ -5481,7 +5481,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.935745+00:00", + "create_time": "2024-06-18 12:45:49.786989+00:00", "table_size": "9 KB" }, "name": "online_page_dimension", @@ -5518,7 +5518,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": true }, @@ -5531,7 +5531,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5544,7 +5544,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5557,7 +5557,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5570,7 +5570,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false }, @@ -5583,7 +5583,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false } @@ -5697,7 +5697,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "create_time": "2024-06-03 12:23:45.941712+00:00", + "create_time": "2024-06-18 12:45:49.791761+00:00", "table_size": "182385 KB" }, "name": "online_sales_fact", @@ -5734,7 +5734,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5747,7 +5747,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5760,7 +5760,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5773,7 +5773,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5786,7 +5786,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5799,7 +5799,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5812,7 +5812,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5825,7 +5825,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5838,7 +5838,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5851,7 +5851,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5864,7 +5864,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5877,7 +5877,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -5890,7 +5890,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5903,7 +5903,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5916,7 +5916,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5929,7 +5929,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5942,7 +5942,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -5955,7 +5955,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -5968,7 +5968,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -5981,7 +5981,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -6052,7 +6052,7 @@ "env": "PROD", "database": "vmart", "cluster_type": "Enterprise", - "cluster_size": "52 GB", + "cluster_size": "243 GB", "subcluster": " ", "communal_storage_path": "" }, @@ -6129,7 +6129,7 @@ "schema": "public", "projection_count": "12", "udx_list": "APPROXIMATE_COUNT_DISTINCT_SYNOPSIS_INFO, APPROXIMATE_MEDIAN, APPROXIMATE_PERCENTILE, AcdDataToCount, AcdDataToLongSyn, AcdDataToSyn, AcdSynToCount, AcdSynToSyn, DelimitedExport, DelimitedExportMulti, EmptyMap, Explode, FAvroParser, FCefParser, FCsvParser, FDelimitedPairParser, FDelimitedParser, FIDXParser, FJSONParser, FRegexParser, FlexTokenizer, JsonExport, JsonExportMulti, KafkaAvroParser, KafkaCheckBrokers, KafkaExport, KafkaInsertDelimiters, KafkaInsertLengths, KafkaJsonParser, KafkaListManyTopics, KafkaListTopics, KafkaOffsets, KafkaParser, KafkaSource, KafkaTopicDetails, MSE, MapAggregate, MapAggregate, MapContainsKey, MapContainsKey, MapContainsValue, MapContainsValue, MapDelimitedExtractor, MapItems, MapItems, MapJSONExtractor, MapKeys, MapKeys, MapKeysInfo, MapKeysInfo, MapLookup, MapLookup, MapLookup, MapPut, MapRegexExtractor, MapSize, MapSize, MapToString, MapToString, MapValues, MapValues, MapValuesOrField, MapVersion, MapVersion, OrcExport, OrcExportMulti, PRC, ParquetExport, ParquetExportMulti, PickBestType, PickBestType, PickBestType, ROC, STV_AsGeoJSON, STV_AsGeoJSON, STV_AsGeoJSON, STV_Create_Index, STV_Create_Index, STV_Create_Index, STV_DWithin, STV_DWithin, STV_DWithin, STV_Describe_Index, STV_Drop_Index, STV_Export2Shapefile, STV_Extent, STV_Extent, STV_ForceLHR, STV_Geography, STV_Geography, STV_GeographyPoint, STV_Geometry, STV_Geometry, STV_GeometryPoint, STV_GeometryPoint, STV_GetExportShapefileDirectory, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_Intersect, STV_IsValidReason, STV_IsValidReason, STV_IsValidReason, STV_LineStringPoint, STV_LineStringPoint, STV_LineStringPoint, STV_MemSize, STV_MemSize, STV_MemSize, STV_NN, STV_NN, STV_NN, STV_PolygonPoint, STV_PolygonPoint, STV_PolygonPoint, STV_Refresh_Index, STV_Refresh_Index, STV_Refresh_Index, STV_Rename_Index, STV_Reverse, STV_SetExportShapefileDirectory, STV_ShpCreateTable, STV_ShpParser, STV_ShpSource, ST_Area, ST_Area, ST_Area, ST_AsBinary, ST_AsBinary, ST_AsBinary, ST_AsText, ST_AsText, ST_AsText, ST_Boundary, ST_Buffer, ST_Centroid, ST_Contains, ST_Contains, ST_Contains, ST_ConvexHull, ST_Crosses, ST_Difference, ST_Disjoint, ST_Disjoint, ST_Disjoint, ST_Distance, ST_Distance, ST_Distance, ST_Envelope, ST_Equals, ST_Equals, ST_Equals, ST_GeoHash, ST_GeoHash, ST_GeoHash, ST_GeographyFromText, ST_GeographyFromWKB, ST_GeomFromGeoHash, ST_GeomFromGeoJSON, ST_GeomFromGeoJSON, ST_GeomFromText, ST_GeomFromText, ST_GeomFromWKB, ST_GeomFromWKB, ST_GeometryN, ST_GeometryN, ST_GeometryN, ST_GeometryType, ST_GeometryType, ST_GeometryType, ST_Intersection, ST_Intersects, ST_Intersects, ST_IsEmpty, ST_IsEmpty, ST_IsEmpty, ST_IsSimple, ST_IsSimple, ST_IsSimple, ST_IsValid, ST_IsValid, ST_IsValid, ST_Length, ST_Length, ST_Length, ST_NumGeometries, ST_NumGeometries, ST_NumGeometries, ST_NumPoints, ST_NumPoints, ST_NumPoints, ST_Overlaps, ST_PointFromGeoHash, ST_PointN, ST_PointN, ST_PointN, ST_Relate, ST_SRID, ST_SRID, ST_SRID, ST_Simplify, ST_SimplifyPreserveTopology, ST_SymDifference, ST_Touches, ST_Touches, ST_Touches, ST_Transform, ST_Union, ST_Union, ST_Within, ST_Within, ST_Within, ST_X, ST_X, ST_X, ST_XMax, ST_XMax, ST_XMax, ST_XMin, ST_XMin, ST_XMin, ST_Y, ST_Y, ST_Y, ST_YMax, ST_YMax, ST_YMax, ST_YMin, ST_YMin, ST_YMin, ST_intersects, SetMapKeys, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_CatCol, Summarize_NumCol, Unnest, VoltageSecureAccess, VoltageSecureAccess, VoltageSecureConfigure, VoltageSecureConfigureGlobal, VoltageSecureProtect, VoltageSecureProtect, VoltageSecureProtectAllKeys, VoltageSecureRefreshPolicy, VoltageSecureVersion, append_centers, apply_bisecting_kmeans, apply_iforest, apply_inverse_pca, apply_inverse_svd, apply_kmeans, apply_kprototypes, apply_normalize, apply_one_hot_encoder, apply_pca, apply_svd, approximate_quantiles, ar_create_blobs, ar_final_newton, ar_save_model, ar_transition_newton, arima_bfgs, arima_line_search, arima_save_model, avg_all_columns_local, bisecting_kmeans_init_model, bk_apply_best_kmeans_results, bk_compute_totss_local, bk_finalize_model, bk_get_rows_in_active_cluster, bk_kmeans_compute_local_centers, bk_kmeans_compute_withinss, bk_kmeans_fast_random_init, bk_kmeans_slow_random_init, bk_kmeanspp_init_cur_cluster, bk_kmeanspp_reset_blob, bk_kmeanspp_select_new_centers, bk_kmeanspp_within_chunk_sum, bk_save_final_model, bk_write_new_cluster_level, blob_to_table, bufUdx, bufUdx, calc_pseudo_centers, calculate_alpha_linear, calculate_hessian_linear1, calculate_hessian_linear2, chi_squared, cleanup_kmeans_files, compute_and_save_global_center, compute_and_save_new_centers, compute_local_totss, compute_local_withinss, compute_new_local_centers, confusion_matrix, coordinate_descent_covariance, corr_matrix, count_rows_in_blob, create_aggregator_blob, error_rate, evaluate_naive_bayes_model, evaluate_reg_model, evaluate_svm_model, export_model_files, finalize_blob_resource_group, get_attr_minmax, get_attr_robust_zscore, get_attr_zscore, get_model_attribute, get_model_summary, get_robust_zscore_median, iforest_create_blobs, iforest_phase0_udf1, iforest_phase0_udf2, iforest_phase1_udf1, iforest_phase1_udf2, iforest_phase1_udf3, iforest_phase1_udf4, iforest_phase2_udf1, iforest_phase2_udf2, iforest_phase2_udf3, iforest_phase2_udf4, iforest_save_model, import_model_files, isOrContains, kmeansAddMetricsToModel, kmeans_init_blobs, kmeans_to_write_final_centers, lift_table, line_search_logistic1, line_search_logistic2, load_rows_into_blocks, map_factor, math_op, matrix_global_xtx, matrix_local_xtx, mode_finder, model_converter, naive_bayes_phase1, naive_bayes_phase1_blob, naive_bayes_phase2, pca_prep1_global, pca_prep1_local, pca_prep2, pmml_parser, predict_arima, predict_autoregressor, predict_linear_reg, predict_logistic_reg, predict_moving_average, predict_naive_bayes, predict_naive_bayes_classes, predict_pmml, predict_poisson_reg, predict_rf_classifier, predict_rf_classifier_classes, predict_rf_regressor, predict_svm_classifier, predict_svm_regressor, predict_xgb_classifier, predict_xgb_classifier_classes, predict_xgb_regressor, random_init, random_init_write, read_from_dfblob, read_map_factor, read_ptree, read_tree, reg_final_bfgs, reg_final_newton, reg_transition_bfgs, reg_transition_newton, reg_write_model, remove_blob, reverse_normalize, rf_blob, rf_clean, rf_phase0_udf1, rf_phase0_udf2, rf_phase1_udf1, rf_phase1_udf2, rf_phase1_udf3, rf_phase1_udf4, rf_phase2_udf1, rf_phase2_udf2, rf_phase2_udf3, rf_phase2_udf4, rf_predictor_importance, rf_save_model, rsquared, save_cv_result, save_pca_model, save_svd_model, save_svm_model, select_new_centers, store_minmax_model, store_one_hot_encoder_model, store_robust_zscore_model, store_zscore_model, table_to_blob, table_to_dfblob, tokenize, topk, update_and_return_sum_of_squared_distances, upgrade_model_format, writeInitialKmeansModelToDfs, xgb_create_blobs, xgb_phase0_udf1, xgb_phase0_udf2, xgb_phase1_udf1, xgb_phase1_udf2, xgb_phase1_udf3, xgb_phase2_udf1, xgb_phase2_udf2, xgb_phase2_udf3, xgb_predictor_importance, xgb_prune, xgb_save_model, yule_walker, ", - "udx_language": "ComplexTypesLib -- Functions for Complex Types | DelimitedExportLib -- Delimited data export package | JsonExportLib -- Json data export package | MachineLearningLib -- Machine learning package | OrcExportLib -- Orc export package | ParquetExportLib -- Parquet export package | ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | KafkaLib -- Kafka streaming load and export | PlaceLib -- Geospatial package | VoltageSecureLib -- Voltage SecureData Connector | TransformFunctions -- User-defined Python library | " + "udx_language": "ApproximateLib -- Approximate package | FlexTableLib -- Flexible Tables Data Load and Query | OrcExportLib -- Orc export package | JsonExportLib -- Json data export package | PlaceLib -- Geospatial package | ParquetExportLib -- Parquet export package | ComplexTypesLib -- Functions for Complex Types | VoltageSecureLib -- Voltage SecureData Connector | KafkaLib -- Kafka streaming load and export | MachineLearningLib -- Machine learning package | DelimitedExportLib -- Delimited data export package | TransformFunctions -- User-defined Python library | " }, "name": "public" } @@ -6304,7 +6304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6317,7 +6317,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -6330,7 +6330,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=18)", + "nativeDataType": "VARCHAR(18)", "recursive": false, "isPartOfKey": false }, @@ -6343,7 +6343,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -6356,7 +6356,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6369,7 +6369,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6382,7 +6382,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6395,7 +6395,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6408,7 +6408,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6421,7 +6421,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6434,7 +6434,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6447,7 +6447,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=9)", + "nativeDataType": "VARCHAR(9)", "recursive": false, "isPartOfKey": false }, @@ -6460,7 +6460,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6473,7 +6473,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -6486,7 +6486,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6499,7 +6499,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -6512,7 +6512,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6525,7 +6525,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6538,7 +6538,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=10)", + "nativeDataType": "VARCHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -6551,7 +6551,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=7)", + "nativeDataType": "CHAR(7)", "recursive": false, "isPartOfKey": false }, @@ -6564,7 +6564,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -6721,7 +6721,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6734,7 +6734,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6747,7 +6747,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -6760,7 +6760,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6773,7 +6773,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6786,7 +6786,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6799,7 +6799,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6812,7 +6812,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6825,7 +6825,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6838,7 +6838,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6851,7 +6851,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6864,7 +6864,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -6877,7 +6877,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6890,7 +6890,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6903,7 +6903,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6916,7 +6916,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6929,7 +6929,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6942,7 +6942,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6955,7 +6955,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6968,7 +6968,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -6981,7 +6981,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -7138,7 +7138,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7151,7 +7151,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -7164,7 +7164,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7177,7 +7177,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7190,7 +7190,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7203,7 +7203,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7216,7 +7216,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7229,7 +7229,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7242,7 +7242,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false }, @@ -7255,7 +7255,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7268,7 +7268,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -7281,7 +7281,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -7438,7 +7438,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7451,7 +7451,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7464,7 +7464,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7477,7 +7477,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7490,7 +7490,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -7503,7 +7503,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7516,7 +7516,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7529,7 +7529,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -7686,7 +7686,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7699,7 +7699,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -7712,7 +7712,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -7725,7 +7725,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -7738,7 +7738,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -7751,7 +7751,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7764,7 +7764,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -7777,7 +7777,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7790,7 +7790,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -7803,7 +7803,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7816,7 +7816,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7829,7 +7829,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7842,7 +7842,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7855,7 +7855,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7868,7 +7868,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -7881,7 +7881,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7894,7 +7894,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7907,7 +7907,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -7920,7 +7920,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -7933,7 +7933,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -7946,7 +7946,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -8103,7 +8103,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8116,7 +8116,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -8129,7 +8129,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -8142,7 +8142,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8155,7 +8155,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -8168,7 +8168,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8181,7 +8181,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8194,7 +8194,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -8207,7 +8207,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -8220,7 +8220,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8233,7 +8233,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -8246,7 +8246,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=32)", + "nativeDataType": "CHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -8259,7 +8259,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -8272,7 +8272,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8285,7 +8285,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8298,7 +8298,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8311,7 +8311,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -8324,7 +8324,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -8481,7 +8481,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8494,7 +8494,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=20)", + "nativeDataType": "VARCHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -8507,7 +8507,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -8520,7 +8520,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=60)", + "nativeDataType": "VARCHAR(60)", "recursive": false, "isPartOfKey": false }, @@ -8533,7 +8533,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -8546,7 +8546,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false } @@ -8703,7 +8703,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8716,7 +8716,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=30)", + "nativeDataType": "CHAR(30)", "recursive": false, "isPartOfKey": false }, @@ -8729,7 +8729,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=10)", + "nativeDataType": "CHAR(10)", "recursive": false, "isPartOfKey": false }, @@ -8742,7 +8742,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false } @@ -8899,7 +8899,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8912,7 +8912,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8925,7 +8925,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8938,7 +8938,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8951,7 +8951,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -8964,7 +8964,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } @@ -9121,7 +9121,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9134,7 +9134,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false }, @@ -9147,7 +9147,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false } @@ -9304,7 +9304,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -9461,7 +9461,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=128)", + "nativeDataType": "VARCHAR(128)", "recursive": false, "isPartOfKey": false } @@ -9709,7 +9709,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9722,7 +9722,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -9735,7 +9735,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9748,7 +9748,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -9761,7 +9761,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -9774,7 +9774,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -9787,7 +9787,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -9800,7 +9800,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -9813,7 +9813,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -9826,7 +9826,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -9839,7 +9839,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9852,7 +9852,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9865,7 +9865,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -9878,7 +9878,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -9891,7 +9891,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9904,7 +9904,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9917,7 +9917,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -9930,7 +9930,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -10087,7 +10087,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10100,7 +10100,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10113,7 +10113,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10126,7 +10126,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10139,7 +10139,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10152,7 +10152,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10165,7 +10165,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10178,7 +10178,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10191,7 +10191,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10204,7 +10204,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10217,7 +10217,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10230,7 +10230,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10243,7 +10243,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -10256,7 +10256,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIME()", + "nativeDataType": "TIME", "recursive": false, "isPartOfKey": false }, @@ -10269,7 +10269,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=8)", + "nativeDataType": "VARCHAR(8)", "recursive": false, "isPartOfKey": false }, @@ -10282,7 +10282,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10295,7 +10295,7 @@ "com.linkedin.pegasus2avro.schema.TimeType": {} } }, - "nativeDataType": "TIMESTAMP_WITH_PRECISION()", + "nativeDataType": "TIMESTAMP", "recursive": false, "isPartOfKey": false } @@ -10452,7 +10452,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10465,7 +10465,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10478,7 +10478,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10491,7 +10491,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10504,7 +10504,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10517,7 +10517,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10530,7 +10530,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10543,7 +10543,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10556,7 +10556,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10569,7 +10569,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10582,7 +10582,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10595,7 +10595,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10608,7 +10608,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=32)", + "nativeDataType": "VARCHAR(32)", "recursive": false, "isPartOfKey": false }, @@ -10621,7 +10621,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10634,7 +10634,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10647,7 +10647,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10660,7 +10660,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10673,7 +10673,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10686,7 +10686,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false } @@ -10934,7 +10934,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10947,7 +10947,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10960,7 +10960,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -10973,7 +10973,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -10986,7 +10986,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false }, @@ -10999,7 +10999,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=100)", + "nativeDataType": "VARCHAR(100)", "recursive": false, "isPartOfKey": false } @@ -11156,7 +11156,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11169,7 +11169,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -11182,7 +11182,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -11195,7 +11195,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -11208,7 +11208,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=50)", + "nativeDataType": "VARCHAR(50)", "recursive": false, "isPartOfKey": false }, @@ -11221,7 +11221,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11234,7 +11234,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=20)", + "nativeDataType": "CHAR(20)", "recursive": false, "isPartOfKey": false }, @@ -11247,7 +11247,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=40)", + "nativeDataType": "VARCHAR(40)", "recursive": false, "isPartOfKey": false }, @@ -11260,7 +11260,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=256)", + "nativeDataType": "VARCHAR(256)", "recursive": false, "isPartOfKey": false }, @@ -11273,7 +11273,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false }, @@ -11286,7 +11286,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "CHAR(length=2)", + "nativeDataType": "CHAR(2)", "recursive": false, "isPartOfKey": false }, @@ -11299,7 +11299,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=64)", + "nativeDataType": "VARCHAR(64)", "recursive": false, "isPartOfKey": false } @@ -11456,7 +11456,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11469,7 +11469,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11482,7 +11482,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11495,7 +11495,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11508,7 +11508,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11521,7 +11521,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11534,7 +11534,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11547,7 +11547,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11560,7 +11560,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11573,7 +11573,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11586,7 +11586,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11599,7 +11599,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "INTEGER()", + "nativeDataType": "INTEGER", "recursive": false, "isPartOfKey": false }, @@ -11612,7 +11612,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11625,7 +11625,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11638,7 +11638,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11651,7 +11651,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11664,7 +11664,7 @@ "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "FLOAT()", + "nativeDataType": "FLOAT", "recursive": false, "isPartOfKey": false }, @@ -11677,7 +11677,7 @@ "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "VARCHAR(length=16)", + "nativeDataType": "VARCHAR(16)", "recursive": false, "isPartOfKey": false }, @@ -11690,7 +11690,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false }, @@ -11703,7 +11703,7 @@ "com.linkedin.pegasus2avro.schema.DateType": {} } }, - "nativeDataType": "DATE()", + "nativeDataType": "DATE", "recursive": false, "isPartOfKey": false } diff --git a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py index 6c719d351c4c20..b080819cea95be 100644 --- a/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py +++ b/metadata-ingestion/tests/unit/utilities/test_sqlalchemy_type_converter.py @@ -1,6 +1,8 @@ from typing import no_type_check +from unittest.mock import MagicMock from sqlalchemy import types +from sqlalchemy.engine.default import DefaultDialect from sqlalchemy_bigquery import STRUCT from datahub.metadata.schema_classes import ( @@ -17,8 +19,11 @@ def test_get_avro_schema_for_sqlalchemy_column(): + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=types.INTEGER() + column_name="test", column_type=types.INTEGER(), inspector=inspector_magic_mock ) assert len(schema_fields) == 1 assert schema_fields[0].fieldPath == "[version=2.0].[type=int].test" @@ -27,7 +32,10 @@ def test_get_avro_schema_for_sqlalchemy_column(): assert schema_fields[0].nullable is True schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=types.String(), nullable=False + column_name="test", + column_type=types.String(), + nullable=False, + inspector=inspector_magic_mock, ) assert len(schema_fields) == 1 assert schema_fields[0].fieldPath == "[version=2.0].[type=string].test" @@ -37,8 +45,13 @@ def test_get_avro_schema_for_sqlalchemy_column(): def test_get_avro_schema_for_sqlalchemy_array_column(): + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=types.ARRAY(types.FLOAT()) + column_name="test", + column_type=types.ARRAY(types.FLOAT()), + inspector=inspector_magic_mock, ) assert len(schema_fields) == 1 assert ( @@ -50,8 +63,13 @@ def test_get_avro_schema_for_sqlalchemy_array_column(): def test_get_avro_schema_for_sqlalchemy_map_column(): + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=MapType(types.String(), types.BOOLEAN()) + column_name="test", + column_type=MapType(types.String(), types.BOOLEAN()), + inspector=inspector_magic_mock, ) assert len(schema_fields) == 1 assert ( @@ -65,9 +83,13 @@ def test_get_avro_schema_for_sqlalchemy_map_column(): def test_get_avro_schema_for_sqlalchemy_struct_column() -> None: + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() schema_fields = get_schema_fields_for_sqlalchemy_column( - column_name="test", column_type=STRUCT(("test", types.INTEGER())) + column_name="test", + column_type=STRUCT(("test", types.INTEGER())), + inspector=inspector_magic_mock, ) assert len(schema_fields) == 2 assert ( @@ -86,7 +108,12 @@ def test_get_avro_schema_for_sqlalchemy_struct_column() -> None: @no_type_check def test_get_avro_schema_for_sqlalchemy_unknown_column(): - schema_fields = get_schema_fields_for_sqlalchemy_column("invalid", "test") + inspector_magic_mock = MagicMock() + inspector_magic_mock.dialect = DefaultDialect() + + schema_fields = get_schema_fields_for_sqlalchemy_column( + "invalid", "test", inspector=inspector_magic_mock + ) assert len(schema_fields) == 1 assert schema_fields[0].type.type == NullTypeClass() assert schema_fields[0].fieldPath == "[version=2.0].[type=null]"