From b5e0833a6f7de076c7bbc0a734f7b11a8c824dde Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:01:22 -0500 Subject: [PATCH 01/17] chore(avro): bump version (#11587) --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index d7fbbb6682e04..79a4ca9384d28 100644 --- a/build.gradle +++ b/build.gradle @@ -107,8 +107,8 @@ project.ext.externalDependency = [ 'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3', 'antlr4': 'org.antlr:antlr4:4.9.3', 'assertJ': 'org.assertj:assertj-core:3.11.1', - 'avro': 'org.apache.avro:avro:1.11.3', - 'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3', + 'avro': 'org.apache.avro:avro:1.11.4', + 'avroCompiler': 'org.apache.avro:avro-compiler:1.11.4', 'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17', 'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3', 'awsS3': 'software.amazon.awssdk:s3:2.26.21', From d94335f6c28c8f016ab294fecaf34fb81338a404 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:10:43 -0500 Subject: [PATCH 02/17] docs(cypress): Update README.txt (#11588) --- smoke-test/tests/cypress/README.txt | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/smoke-test/tests/cypress/README.txt b/smoke-test/tests/cypress/README.txt index 1210e08b861e0..edd40203fe55e 100644 --- a/smoke-test/tests/cypress/README.txt +++ b/smoke-test/tests/cypress/README.txt @@ -1,18 +1,7 @@ # Quick Run Tests with UI -cd smoke-test/ -./cypress-dev.sh - -# Running Cypress Tests Locally - -1. Make sure the packages are installed. It uses some node modules to run locally. Run `yarn install` from this directory. If you don't have `yarn`, download it. - -2. Cypress tests run against your local deployment of datahub. They are dependent on the data inside. There is sample cypress data in this directory. Ideally you want to delete all the data in your local instance before ingesting the cypress data as it will throw off a few tests (like the search tests) if you have data cypress is not expecting. However, most tests will still pass- most visit specific entity pages. - -3. Ingest the cypress data! Using datahub cli, run `datahub ingest -c example_to_datahub_rest.yml` and then `datahub ingest -c example_siblings_to_datahub_rest.yml`. - -4. Set the port that you want to run your cypress tests against in ./cypress-config.json. The default is 9002- if you are developing on react locally, you probably want 3000. Do not commit this change to github. - -5. Now, start the local cypress server: `npx cypress open`. - +1. Run quickstart + $ ./gradlew quickstartDebug +2. Load Cypress fixture data and run Cypress UI + $ ./gradlew cypressDev From 9ffd2dc8caf12a9a538352758c7d8bc132b5f604 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 10 Oct 2024 17:57:23 -0500 Subject: [PATCH 03/17] fix(trivy): multi-repo, bump trivy version (#11590) --- .github/workflows/docker-unified.yml | 30 ++++++++++++++++++---------- docker/build.gradle | 2 ++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index d1c16b567158a..ba7b54287ad25 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -186,9 +186,10 @@ jobs: with: image: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -250,9 +251,10 @@ jobs: with: image: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -314,9 +316,10 @@ jobs: with: image: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -378,9 +381,10 @@ jobs: with: image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -444,9 +448,10 @@ jobs: with: image: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -498,9 +503,10 @@ jobs: with: image: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -552,9 +558,10 @@ jobs: with: image: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -606,9 +613,10 @@ jobs: with: image: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -808,9 +816,10 @@ jobs: with: image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} - name: Run Trivy vulnerability scanner Slim Image - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} format: "template" @@ -896,9 +905,10 @@ jobs: with: image: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} - name: Run Trivy vulnerability scanner Full Image - uses: aquasecurity/trivy-action@0.25.0 + uses: aquasecurity/trivy-action@0.26.0 env: TRIVY_OFFLINE_SCAN: true + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 with: image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} format: "template" diff --git a/docker/build.gradle b/docker/build.gradle index c09bf16d1d724..47f52079e67e0 100644 --- a/docker/build.gradle +++ b/docker/build.gradle @@ -31,6 +31,8 @@ ext { ] // Postgres pg_quickstart_modules = quickstart_modules - [':docker:mysql-setup'] + [':docker:postgres-setup'] + + revision = 1 // increment to trigger rebuild } tasks.register('minDockerCompose2.20', Exec) { From 2205920aaa57e787efb2be9f052e0d8895d484a7 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:07:17 -0500 Subject: [PATCH 04/17] fix(trivy): also add alternative java db (#11591) --- .github/workflows/docker-unified.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml index ba7b54287ad25..ef5770ccb167a 100644 --- a/.github/workflows/docker-unified.yml +++ b/.github/workflows/docker-unified.yml @@ -190,6 +190,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_GMS_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -255,6 +256,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_MAE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -320,6 +322,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_MCE_CONSUMER_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -385,6 +388,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -452,6 +456,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_FRONTEND_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -507,6 +512,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_KAFKA_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -562,6 +568,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_MYSQL_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -617,6 +624,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_ELASTIC_SETUP_IMAGE }}:${{ needs.setup.outputs.unique_tag }} format: "template" @@ -820,6 +828,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_slim_build.outputs.tag }} format: "template" @@ -909,6 +918,7 @@ jobs: env: TRIVY_OFFLINE_SCAN: true TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2,ghcr.io/aquasecurity/trivy-db:2 + TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1,ghcr.io/aquasecurity/trivy-java-db:1 with: image-ref: ${{ env.DATAHUB_INGESTION_IMAGE }}:${{ needs.datahub_ingestion_full_build.outputs.tag }} format: "template" From f43720e4ec90dcad5cd2dbf3fa82653e0627d53d Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:16:33 -0500 Subject: [PATCH 05/17] feat(openapi-v3): generic entities scroll (#11564) --- .../metadata/search/SearchService.java | 5 +- .../v3/models/GenericEntityAspectsBodyV3.java | 15 + .../controller/GenericEntitiesController.java | 107 ++++--- .../v2/controller/EntityController.java | 24 +- .../openapi/v3/OpenAPIV3Generator.java | 266 +++++++++++++++++- .../v3/controller/EntityController.java | 142 +++++++++- 6 files changed, 492 insertions(+), 67 deletions(-) create mode 100644 metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityAspectsBodyV3.java diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java index 1cd738656d972..ecded1bb9c384 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java @@ -13,6 +13,7 @@ import com.linkedin.metadata.utils.metrics.MetricUtils; import io.datahubproject.metadata.context.OperationContext; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -213,7 +214,7 @@ public SearchResult searchAcrossEntities( * @return some entities to search */ public List getEntitiesToSearch( - @Nonnull OperationContext opContext, @Nonnull List inputEntities, int size) { + @Nonnull OperationContext opContext, @Nonnull Collection inputEntities, int size) { List nonEmptyEntities; List lowercaseEntities = inputEntities.stream().map(String::toLowerCase).collect(Collectors.toList()); @@ -247,7 +248,7 @@ public List getEntitiesToSearch( @Nonnull public ScrollResult scrollAcrossEntities( @Nonnull OperationContext opContext, - @Nonnull List entities, + @Nonnull Collection entities, @Nonnull String input, @Nullable Filter postFilters, List sortCriteria, diff --git a/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityAspectsBodyV3.java b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityAspectsBodyV3.java new file mode 100644 index 0000000000000..27107bae87d0e --- /dev/null +++ b/metadata-service/openapi-servlet/models/src/main/java/io/datahubproject/openapi/v3/models/GenericEntityAspectsBodyV3.java @@ -0,0 +1,15 @@ +package io.datahubproject.openapi.v3.models; + +import java.util.Set; +import javax.annotation.Nullable; +import lombok.Builder; +import lombok.Data; +import lombok.extern.jackson.Jacksonized; + +@Data +@Jacksonized +@Builder +public class GenericEntityAspectsBodyV3 { + @Nullable private Set entities; + @Nullable private Set aspects; +} diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index b7f52e61e9244..7e7929e7f27d3 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -97,6 +97,7 @@ public abstract class GenericEntitiesController< * @param aspectNames the aspect names present * @param withSystemMetadata whether to include system metadata in the result * @param scrollId the pagination token + * @param expandEmpty whether to expand an empty aspects list to all aspects * @return result containing entities/aspects * @throws URISyntaxException parsing error */ @@ -105,14 +106,16 @@ protected abstract S buildScrollResult( SearchEntityArray searchEntities, Set aspectNames, boolean withSystemMetadata, - @Nullable String scrollId) + @Nullable String scrollId, + boolean expandEmpty) throws URISyntaxException; protected List buildEntityList( @Nonnull OperationContext opContext, List urns, - Set aspectNames, - boolean withSystemMetadata) + @Nullable Set aspectNames, + boolean withSystemMetadata, + boolean expandEmpty) throws URISyntaxException { LinkedHashMap> versionMap = @@ -122,7 +125,7 @@ protected List buildEntityList( urn -> Map.entry( urn, - aspectNames.stream() + Optional.ofNullable(aspectNames).orElse(Set.of()).stream() .map(aspectName -> Map.entry(aspectName, 0L)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))) .collect( @@ -133,14 +136,30 @@ protected List buildEntityList( throw new IllegalStateException("Duplicate key"); }, LinkedHashMap::new)), - 0L); - return buildEntityVersionedAspectList(opContext, versionMap, withSystemMetadata); + 0L, + expandEmpty); + + return buildEntityVersionedAspectList( + opContext, urns, versionMap, withSystemMetadata, expandEmpty); } + /** + * Build a list of entities for an API response + * + * @param opContext the operation context + * @param requestedUrns list of urns requested + * @param fetchUrnAspectVersions the map of urn to aspect name and version to fetch + * @param withSystemMetadata whether to include system metadata in the response entity + * @param expandEmpty whether to expand an empty aspects list to all aspects + * @return entity responses + * @throws URISyntaxException urn parsing error + */ protected abstract List buildEntityVersionedAspectList( @Nonnull OperationContext opContext, - LinkedHashMap> urnAspectVersions, - boolean withSystemMetadata) + Collection requestedUrns, + LinkedHashMap> fetchUrnAspectVersions, + boolean withSystemMetadata, + boolean expandEmpty) throws URISyntaxException; protected abstract List buildEntityList( @@ -225,13 +244,17 @@ public ResponseEntity getEntities( authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); } + Set mergedAspects = + ImmutableSet.builder().addAll(aspects1).addAll(aspects2).build(); + return ResponseEntity.ok( buildScrollResult( opContext, result.getEntities(), - ImmutableSet.builder().addAll(aspects1).addAll(aspects2).build(), + mergedAspects, withSystemMetadata, - result.getScrollId())); + result.getScrollId(), + true)); } @Tag(name = "Generic Entities") @@ -269,7 +292,8 @@ public ResponseEntity getEntity( opContext, List.of(urn), ImmutableSet.builder().addAll(aspects1).addAll(aspects2).build(), - withSystemMetadata) + withSystemMetadata, + true) .stream() .findFirst() .map(ResponseEntity::ok) @@ -344,13 +368,16 @@ public ResponseEntity getAspect( final List resultList; if (version == 0) { - resultList = buildEntityList(opContext, List.of(urn), Set.of(aspectName), withSystemMetadata); + resultList = + buildEntityList(opContext, List.of(urn), Set.of(aspectName), withSystemMetadata, true); } else { resultList = buildEntityVersionedAspectList( opContext, + List.of(urn), new LinkedHashMap<>(Map.of(urn, Map.of(aspectName, version))), - withSystemMetadata); + withSystemMetadata, + true); } return resultList.stream() @@ -395,9 +422,10 @@ public ResponseEntity headAspect( authentication.getActor().toUrnStr() + " is unauthorized to " + EXISTS + " entities."); } - return exists(opContext, urn, lookupAspectSpec(urn, aspectName).getName(), includeSoftDelete) - ? ResponseEntity.noContent().build() - : ResponseEntity.notFound().build(); + return lookupAspectSpec(urn, aspectName) + .filter(aspectSpec -> exists(opContext, urn, aspectSpec.getName(), includeSoftDelete)) + .map(aspectSpec -> ResponseEntity.noContent().build()) + .orElse(ResponseEntity.notFound().build()); } @Tag(name = "Generic Entities") @@ -443,7 +471,7 @@ public void deleteEntity( entityService.deleteUrn(opContext, urn); } else { aspects.stream() - .map(aspectName -> lookupAspectSpec(urn, aspectName).getName()) + .map(aspectName -> lookupAspectSpec(urn, aspectName).get().getName()) .forEach( aspectName -> entityService.deleteAspect(opContext, entityUrn, aspectName, Map.of(), true)); @@ -515,8 +543,11 @@ public void deleteAspect( authentication.getActor().toUrnStr() + " is unauthorized to " + DELETE + " entities."); } - entityService.deleteAspect( - opContext, entityUrn, lookupAspectSpec(urn, aspectName).getName(), Map.of(), true); + lookupAspectSpec(urn, aspectName) + .ifPresent( + aspectSpec -> + entityService.deleteAspect( + opContext, entityUrn, aspectSpec.getName(), Map.of(), true)); } @Tag(name = "Generic Aspects") @@ -554,7 +585,7 @@ public ResponseEntity createAspect( authentication.getActor().toUrnStr() + " is unauthorized to " + CREATE + " entities."); } - AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName); + AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName).get(); ChangeMCP upsert = toUpsertItem( opContext.getRetrieverContext().get().getAspectRetriever(), @@ -618,7 +649,7 @@ public ResponseEntity patchAspect( authentication.getActor().toUrnStr() + " is unauthorized to " + UPDATE + " entities."); } - AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName); + AspectSpec aspectSpec = lookupAspectSpec(entitySpec, aspectName).get(); RecordTemplate currentValue = entityService.getAspect(opContext, urn, aspectSpec.getName(), 0); GenericPatchTemplate genericPatchTemplate = @@ -672,15 +703,18 @@ protected Boolean exists( * * @param requestedAspectNames requested aspects * @param map values + * @param expandEmpty whether to expand empty aspect names to all aspect names * @return updated map */ protected LinkedHashMap> resolveAspectNames( - LinkedHashMap> requestedAspectNames, @Nonnull T defaultValue) { + LinkedHashMap> requestedAspectNames, + @Nonnull T defaultValue, + boolean expandEmpty) { return requestedAspectNames.entrySet().stream() .map( entry -> { final Urn urn = entry.getKey(); - if (entry.getValue().isEmpty() || entry.getValue().containsKey("")) { + if (expandEmpty && (entry.getValue().isEmpty() || entry.getValue().containsKey(""))) { // All aspects specified Set allNames = entityRegistry.getEntitySpec(urn.getEntityType()).getAspectSpecs().stream() @@ -694,15 +728,16 @@ protected LinkedHashMap> resolveAspectNames( Map.entry( aspectName, entry.getValue().getOrDefault("", defaultValue))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); - } else { + } else if (!entry.getValue().keySet().isEmpty()) { final Map normalizedNames = entry.getValue().keySet().stream() .map( requestAspectName -> Map.entry( - requestAspectName, - lookupAspectSpec(urn, requestAspectName).getName())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + requestAspectName, lookupAspectSpec(urn, requestAspectName))) + .filter(aspectSpecEntry -> aspectSpecEntry.getValue().isPresent()) + .collect( + Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get().getName())); return Map.entry( urn, entry.getValue().entrySet().stream() @@ -712,8 +747,11 @@ protected LinkedHashMap> resolveAspectNames( Map.entry( normalizedNames.get(reqEntry.getKey()), reqEntry.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } else { + return (Map.Entry>) null; } }) + .filter(Objects::nonNull) .collect( Collectors.toMap( Map.Entry::getKey, @@ -732,12 +770,12 @@ protected Map> toAspectMap( Map.entry( a.getName(), Pair.of( - toRecordTemplate(lookupAspectSpec(urn, a.getName()), a), + toRecordTemplate(lookupAspectSpec(urn, a.getName()).get(), a), withSystemMetadata ? a.getSystemMetadata() : null))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - protected AspectSpec lookupAspectSpec(Urn urn, String aspectName) { + protected Optional lookupAspectSpec(Urn urn, String aspectName) { return lookupAspectSpec(entityRegistry.getEntitySpec(urn.getEntityType()), aspectName); } @@ -777,13 +815,16 @@ protected ChangeMCP toUpsertItem( * * @return */ - protected static AspectSpec lookupAspectSpec(EntitySpec entitySpec, String aspectName) { + protected static Optional lookupAspectSpec(EntitySpec entitySpec, String aspectName) { + if (entitySpec == null) { + return Optional.empty(); + } + return entitySpec.getAspectSpec(aspectName) != null - ? entitySpec.getAspectSpec(aspectName) + ? Optional.of(entitySpec.getAspectSpec(aspectName)) : entitySpec.getAspectSpecs().stream() .filter(aspec -> aspec.getName().toLowerCase().equals(aspectName)) - .findFirst() - .get(); + .findFirst(); } protected static Urn validatedUrn(String urn) throws InvalidUrnException { diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java index d20acbee79b22..28537b849b68a 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java @@ -46,6 +46,7 @@ import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -114,7 +115,8 @@ public ResponseEntity> g opContext, urns, new HashSet<>(request.getAspectNames()), - request.isWithSystemMetadata()))) + request.isWithSystemMetadata(), + true))) .build())); } @@ -124,10 +126,12 @@ public GenericEntityScrollResultV2 buildScrollResult( SearchEntityArray searchEntities, Set aspectNames, boolean withSystemMetadata, - @Nullable String scrollId) + @Nullable String scrollId, + boolean expandEmpty) throws URISyntaxException { return GenericEntityScrollResultV2.builder() - .results(toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata)) + .results( + toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata, true)) .scrollId(scrollId) .build(); } @@ -155,7 +159,7 @@ protected AspectsBatch toMCPBatch( while (aspectItr.hasNext()) { Map.Entry aspect = aspectItr.next(); - AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()); + AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()).get(); if (aspectSpec != null) { ChangeItemImpl.ChangeItemImplBuilder builder = @@ -192,12 +196,14 @@ protected AspectsBatch toMCPBatch( @Override protected List buildEntityVersionedAspectList( @Nonnull OperationContext opContext, + Collection requestedUrns, LinkedHashMap> urnAspectVersions, - boolean withSystemMetadata) + boolean withSystemMetadata, + boolean expandEmpty) throws URISyntaxException { Map> aspects = entityService.getEnvelopedVersionedAspects( - opContext, resolveAspectNames(urnAspectVersions, 0L), true); + opContext, resolveAspectNames(urnAspectVersions, 0L, true), true); return urnAspectVersions.keySet().stream() .map( @@ -230,13 +236,15 @@ private List toRecordTemplates( @Nonnull OperationContext opContext, SearchEntityArray searchEntities, Set aspectNames, - boolean withSystemMetadata) + boolean withSystemMetadata, + boolean expandEmpty) throws URISyntaxException { return buildEntityList( opContext, searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()), aspectNames, - withSystemMetadata); + withSystemMetadata, + true); } @Override diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java index 1b3dcfc94f4f2..e33ad24a6c248 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/OpenAPIV3Generator.java @@ -60,6 +60,8 @@ public class OpenAPIV3Generator { private static final String ENTITY_REQUEST_SUFFIX = "Entity" + REQUEST_SUFFIX; private static final String ENTITY_RESPONSE_SUFFIX = "Entity" + RESPONSE_SUFFIX; private static final String NAME_SKIP_CACHE = "skipCache"; + private static final String ASPECTS = "Aspects"; + private static final String ENTITIES = "Entities"; public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { final Set aspectNames = entityRegistry.getAspectSpecs().keySet(); @@ -75,10 +77,19 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { info.setTitle("Entity API"); info.setDescription("This is a service for DataHub Entities."); info.setVersion("v3"); + // Components final Components components = new Components(); + + // Cross-entity components + components.addSchemas( + ENTITIES + ENTITY_REQUEST_SUFFIX, buildEntitiesRequestSchema(entityRegistry, aspectNames)); + components.addSchemas( + ENTITIES + ENTITY_RESPONSE_SUFFIX, buildEntitySchema(entityRegistry, aspectNames, true)); + components.addSchemas( + "Scroll" + ENTITIES + ENTITY_RESPONSE_SUFFIX, buildEntitiesScrollSchema()); + // --> Aspect components - components.addSchemas("SortOrder", new Schema()._enum(List.of("ASCENDING", "DESCENDING"))); components.addSchemas("AspectPatch", buildAspectPatchSchema()); components.addSchemas( "BatchGetRequestBody", @@ -94,6 +105,10 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { .description("System headers for the operation.") .nullable(true))) .nullable(true)); + + // --> Aspect components + components.addSchemas( + ASPECTS + ASPECT_RESPONSE_SUFFIX, buildAspectsRefResponseSchema(entityRegistry)); entityRegistry .getAspectSpecs() .values() @@ -108,6 +123,7 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { upperAspectName + ASPECT_RESPONSE_SUFFIX, buildAspectRefResponseSchema(upperAspectName)); }); + // --> Entity components entityRegistry.getEntitySpecs().values().stream() .filter(e -> aspectNames.contains(e.getKeyAspectName())) @@ -124,18 +140,34 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { "BatchGet" + entityName + ENTITY_REQUEST_SUFFIX, buildEntityBatchGetRequestSchema(e, aspectNames)); }); + + components.addSchemas("SortOrder", new Schema()._enum(List.of("ASCENDING", "DESCENDING"))); + // TODO: Correct handling of SystemMetadata and AuditStamp + components.addSchemas( + "SystemMetadata", new Schema().type(TYPE_OBJECT).additionalProperties(true)); + components.addSchemas("AuditStamp", new Schema().type(TYPE_OBJECT).additionalProperties(true)); + // Parameters + + // --> Entity Parameters entityRegistry.getEntitySpecs().values().stream() .filter(e -> definitionNames.contains(e.getKeyAspectName())) .forEach( e -> { - final String parameterName = toUpperFirst(e.getName()) + "Aspects"; + final String parameterName = toUpperFirst(e.getName()) + ASPECTS; components.addParameters( parameterName + MODEL_VERSION, buildParameterSchema(e, definitionNames)); }); + addExtraParameters(components); + // Path final Paths paths = new Paths(); + + // --> Cross-entity Paths + paths.addPathItem("/v3/entity/scroll", buildGenericListEntitiesPath()); + + // --> Entity Paths entityRegistry.getEntitySpecs().values().stream() .filter(e -> definitionNames.contains(e.getName())) .sorted(Comparator.comparing(EntitySpec::getName)) @@ -151,6 +183,8 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { String.format("/v3/entity/%s/{urn}", e.getName().toLowerCase()), buildSingleEntityPath(e)); }); + + // --> Aspect Paths entityRegistry.getEntitySpecs().values().stream() .filter(e -> definitionNames.contains(e.getName())) .sorted(Comparator.comparing(EntitySpec::getName)) @@ -168,16 +202,12 @@ public static OpenAPI generateOpenApiSpec(EntityRegistry entityRegistry) { buildSingleEntityAspectPath( e, a.getName(), a.getPegasusSchema().getName()))); }); - // TODO: Correct handling of SystemMetadata and AuditStamp - components.addSchemas( - "SystemMetadata", new Schema().type(TYPE_OBJECT).additionalProperties(true)); - components.addSchemas("AuditStamp", new Schema().type(TYPE_OBJECT).additionalProperties(true)); return new OpenAPI().openapi("3.0.1").info(info).paths(paths).components(components); } private static PathItem buildSingleEntityPath(final EntitySpec entity) { final String upperFirst = toUpperFirst(entity.getName()); - final String aspectParameterName = upperFirst + "Aspects"; + final String aspectParameterName = upperFirst + ASPECTS; final PathItem result = new PathItem(); // Get Operation @@ -280,7 +310,7 @@ private static PathItem buildSingleEntityPath(final EntitySpec entity) { private static PathItem buildListEntityPath(final EntitySpec entity) { final String upperFirst = toUpperFirst(entity.getName()); - final String aspectParameterName = upperFirst + "Aspects"; + final String aspectParameterName = upperFirst + ASPECTS; final PathItem result = new PathItem(); final List parameters = List.of( @@ -327,7 +357,8 @@ private static PathItem buildListEntityPath(final EntitySpec entity) { .summary(String.format("Scroll/List %s.", upperFirst)) .parameters(parameters) .tags(List.of(entity.getName() + " Entity")) - .description("Scroll indexed entities. Will not include soft deleted entities.") + .description( + "Scroll indexed entities. Will not include soft deleted entities by default.") .responses(new ApiResponses().addApiResponse("200", successApiResponse))); // Post Operation @@ -452,6 +483,74 @@ private static PathItem buildBatchGetEntityPath(final EntitySpec entity) { return result; } + private static PathItem buildGenericListEntitiesPath() { + final PathItem result = new PathItem(); + final List parameters = + List.of( + new Parameter() + .in(NAME_QUERY) + .name(NAME_SYSTEM_METADATA) + .description("Include systemMetadata with response.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_INCLUDE_SOFT_DELETE) + .description("Include soft-deleted aspects with response.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter() + .in(NAME_QUERY) + .name(NAME_SKIP_CACHE) + .description("Skip cache when listing entities.") + .schema(new Schema().type(TYPE_BOOLEAN)._default(false)), + new Parameter().$ref("#/components/parameters/PaginationCount" + MODEL_VERSION), + new Parameter().$ref("#/components/parameters/ScrollId" + MODEL_VERSION), + new Parameter().$ref("#/components/parameters/SortBy" + MODEL_VERSION), + new Parameter().$ref("#/components/parameters/SortOrder" + MODEL_VERSION), + new Parameter().$ref("#/components/parameters/ScrollQuery" + MODEL_VERSION)); + final ApiResponse successApiResponse = + new ApiResponse() + .description("Success") + .content( + new Content() + .addMediaType( + "application/json", + new MediaType() + .schema( + new Schema() + .$ref( + String.format( + "#/components/schemas/Scroll%s%s", + ENTITIES, ENTITY_RESPONSE_SUFFIX))))); + + final RequestBody requestBody = + new RequestBody() + .description( + "Scroll entities and aspects. If the `aspects` list is not specified then NO aspects will be returned. If the `aspects` list is emtpy, all aspects will be returned.") + .required(false) + .content( + new Content() + .addMediaType( + "application/json", + new MediaType() + .schema( + new Schema() + .$ref( + String.format( + "#/components/schemas/%s%s", + ENTITIES, ENTITY_REQUEST_SUFFIX))))); + + result.setPost( + new Operation() + .summary(String.format("Scroll/List %s.", ENTITIES)) + .parameters(parameters) + .tags(List.of("Generic Entities")) + .description("Scroll indexed entities. Will not include soft deleted entities.") + .requestBody(requestBody) + .responses(new ApiResponses().addApiResponse("200", successApiResponse))); + + return result; + } + private static void addExtraParameters(final Components components) { components.addParameters( "ScrollId" + MODEL_VERSION, @@ -499,7 +598,8 @@ private static void addExtraParameters(final Components components) { new Parameter() .in(NAME_QUERY) .name(NAME_QUERY) - .description("Structured search query.") + .description( + "Structured search query. See Elasticsearch documentation on `query_string` syntax.") .example("*") .schema(new Schema().type(TYPE_STRING)._default("*"))); } @@ -528,6 +628,7 @@ private static Parameter buildParameterSchema( .name("aspects") .explode(true) .description("Aspects to include.") + .required(false) .example(aspectNames) .schema(schema); } @@ -582,6 +683,43 @@ private static void addAspectSchemas(final Components components, final AspectSp } } + /** + * Generate schema for cross-entity scroll/list response + * + * @param entityRegistry entity registry + * @return schema + */ + private static Schema buildAspectsRefResponseSchema(final EntityRegistry entityRegistry) { + final Schema result = + new Schema<>() + .type(TYPE_OBJECT) + .description(ASPECT_DESCRIPTION) + .required(List.of(PROPERTY_VALUE)); + + entityRegistry + .getAspectSpecs() + .values() + .forEach( + aspect -> + result.addProperty( + PROPERTY_VALUE, new Schema<>().$ref(PATH_DEFINITIONS + aspect.getName()))); + result.addProperty( + NAME_SYSTEM_METADATA, + new Schema<>() + .type(TYPE_OBJECT) + .anyOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "SystemMetadata"))) + .description("System metadata for the aspect.") + .nullable(true)); + result.addProperty( + NAME_AUDIT_STAMP, + new Schema<>() + .type(TYPE_OBJECT) + .anyOf(List.of(new Schema().$ref(PATH_DEFINITIONS + "AuditStamp"))) + .description("Audit stamp for the aspect.") + .nullable(true)); + return result; + } + private static Schema buildAspectRefResponseSchema(final String aspectName) { final Schema result = new Schema<>() @@ -612,7 +750,8 @@ private static Schema buildAspectRefRequestSchema(final String aspectName) { .type(TYPE_OBJECT) .description(ASPECT_DESCRIPTION) .required(List.of(PROPERTY_VALUE)) - .addProperty(PROPERTY_VALUE, new Schema<>().$ref(PATH_DEFINITIONS + aspectName)); + .addProperty( + PROPERTY_VALUE, new Schema<>().$ref(PATH_DEFINITIONS + toUpperFirst(aspectName))); result.addProperty( NAME_SYSTEM_METADATA, new Schema<>() @@ -657,6 +796,111 @@ private static Schema buildEntitySchema( .properties(properties); } + /** + * Generate cross-entity schema + * + * @param entityRegistry entity registry + * @param withSystemMetadata include system metadata + * @return schema + */ + private static Schema buildEntitySchema( + final EntityRegistry entityRegistry, + final Set aspectNames, + final boolean withSystemMetadata) { + final Map properties = + entityRegistry.getAspectSpecs().entrySet().stream() + .filter(a -> aspectNames.contains(a.getValue().getName())) + .collect( + Collectors.toMap( + Map.Entry::getKey, + a -> + buildAspectRef( + a.getValue().getPegasusSchema().getName(), withSystemMetadata))); + properties.put( + PROPERTY_URN, new Schema<>().type(TYPE_STRING).description("Unique id for " + ENTITIES)); + + return new Schema<>() + .type(TYPE_OBJECT) + .description(ENTITIES + " object.") + .required(List.of(PROPERTY_URN)) + .properties(properties); + } + + /** + * Generate cross-entity schema + * + * @param entityRegistry entity registry + * @param definitionNames include aspects + * @return schema + */ + private static Schema buildEntitiesRequestSchema( + final EntityRegistry entityRegistry, final Set definitionNames) { + + final Set keyAspects = new HashSet<>(); + + final List entityNames = + entityRegistry.getEntitySpecs().values().stream() + .peek(entitySpec -> keyAspects.add(entitySpec.getKeyAspectName())) + .map(EntitySpec::getName) + .sorted() + .toList(); + + Schema entitiesSchema = + new Schema().type(TYPE_ARRAY).items(new Schema().type(TYPE_STRING)._enum(entityNames)); + + final List aspectNames = + entityRegistry.getAspectSpecs().values().stream() + .filter(aspectSpec -> !aspectSpec.isTimeseries()) + .map(AspectSpec::getName) + .filter(definitionNames::contains) // Only if aspect is defined + .distinct() + .sorted() + .collect(Collectors.toList()); + + Schema aspectsSchema = + new Schema().type(TYPE_ARRAY).items(new Schema().type(TYPE_STRING)._enum(aspectNames)); + + return new Schema<>() + .type(TYPE_OBJECT) + .description(ENTITIES + " request object.") + .example( + Map.of( + "entities", entityNames.stream().filter(n -> !n.startsWith("dataHub")).toList(), + "aspects", + aspectNames.stream() + .filter(n -> !n.startsWith("dataHub") && !keyAspects.contains(n)) + .toList())) + .properties( + Map.of( + "entities", entitiesSchema, + "aspects", aspectsSchema)); + } + + /** + * Generate schema for cross-entity scroll/list response + * + * @return schema + */ + private static Schema buildEntitiesScrollSchema() { + return new Schema<>() + .type(TYPE_OBJECT) + .description("Scroll across (list) " + ENTITIES + " objects.") + .required(List.of("entities")) + .addProperty( + NAME_SCROLL_ID, + new Schema<>().type(TYPE_STRING).description("Scroll id for pagination.")) + .addProperty( + "entities", + new Schema<>() + .type(TYPE_ARRAY) + .description(ENTITIES + " object.") + .items( + new Schema<>() + .$ref( + String.format( + "#/components/schemas/%s%s", ENTITIES, ENTITY_RESPONSE_SUFFIX)))); + } + private static Schema buildEntityScrollSchema(final EntitySpec entity) { return new Schema<>() .type(TYPE_OBJECT) diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index d7694f3aed933..c7d8c72f8a1c3 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -25,10 +25,15 @@ import com.linkedin.metadata.entity.ebean.batch.AspectsBatchImpl; import com.linkedin.metadata.entity.ebean.batch.ChangeItemImpl; import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.ScrollResult; import com.linkedin.metadata.search.SearchEntity; import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.metadata.utils.SearchUtil; import com.linkedin.mxe.SystemMetadata; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.RequestContext; @@ -37,6 +42,7 @@ import io.datahubproject.openapi.exception.UnauthorizedException; import io.datahubproject.openapi.v3.models.AspectItem; import io.datahubproject.openapi.v3.models.GenericAspectV3; +import io.datahubproject.openapi.v3.models.GenericEntityAspectsBodyV3; import io.datahubproject.openapi.v3.models.GenericEntityScrollResultV3; import io.datahubproject.openapi.v3.models.GenericEntityV3; import io.swagger.v3.oas.annotations.Hidden; @@ -45,6 +51,9 @@ import jakarta.servlet.http.HttpServletRequest; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -60,6 +69,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.util.CollectionUtils; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -109,7 +119,101 @@ public ResponseEntity> getEntityBatch( } return ResponseEntity.of( - Optional.of(buildEntityVersionedAspectList(opContext, requestMap, withSystemMetadata))); + Optional.of( + buildEntityVersionedAspectList( + opContext, requestMap.keySet(), requestMap, withSystemMetadata, true))); + } + + @Tag(name = "Generic Entities", description = "API for interacting with generic entities.") + @PostMapping(value = "/scroll", produces = MediaType.APPLICATION_JSON_VALUE) + @Operation(summary = "Scroll entities") + public ResponseEntity scrollEntities( + HttpServletRequest request, + @RequestParam(value = "count", defaultValue = "10") Integer count, + @RequestParam(value = "query", defaultValue = "*") String query, + @RequestParam(value = "scrollId", required = false) String scrollId, + @RequestParam(value = "sort", required = false, defaultValue = "urn") String sortField, + @RequestParam(value = "sortCriteria", required = false) List sortFields, + @RequestParam(value = "sortOrder", required = false, defaultValue = "ASCENDING") + String sortOrder, + @RequestParam(value = "systemMetadata", required = false, defaultValue = "false") + Boolean withSystemMetadata, + @RequestParam(value = "skipCache", required = false, defaultValue = "false") + Boolean skipCache, + @RequestParam(value = "includeSoftDelete", required = false, defaultValue = "false") + Boolean includeSoftDelete, + @RequestBody @Nonnull GenericEntityAspectsBodyV3 entityAspectsBody) + throws URISyntaxException { + + final Collection resolvedEntityNames; + if (entityAspectsBody.getEntities() != null) { + resolvedEntityNames = + entityAspectsBody.getEntities().stream() + .map(entityName -> entityRegistry.getEntitySpec(entityName)) + .map(EntitySpec::getName) + .toList(); + } else { + resolvedEntityNames = + entityRegistry.getEntitySpecs().values().stream().map(EntitySpec::getName).toList(); + } + + Authentication authentication = AuthenticationContext.getAuthentication(); + + OperationContext opContext = + OperationContext.asSession( + systemOperationContext, + RequestContext.builder() + .buildOpenapi( + authentication.getActor().toUrnStr(), + request, + "scrollEntities", + resolvedEntityNames), + authorizationChain, + authentication, + true); + + if (!AuthUtil.isAPIAuthorizedEntityType(opContext, READ, resolvedEntityNames)) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); + } + + List sortCriteria; + if (!CollectionUtils.isEmpty(sortFields)) { + sortCriteria = new ArrayList<>(); + sortFields.forEach( + field -> sortCriteria.add(SearchUtil.sortBy(field, SortOrder.valueOf(sortOrder)))); + } else { + sortCriteria = + Collections.singletonList(SearchUtil.sortBy(sortField, SortOrder.valueOf(sortOrder))); + } + + ScrollResult result = + searchService.scrollAcrossEntities( + opContext + .withSearchFlags(flags -> DEFAULT_SEARCH_FLAGS) + .withSearchFlags(flags -> flags.setSkipCache(skipCache)) + .withSearchFlags(flags -> flags.setIncludeSoftDeleted(includeSoftDelete)), + resolvedEntityNames, + query, + null, + sortCriteria, + scrollId, + null, + count); + + if (!AuthUtil.isAPIAuthorizedResult(opContext, result)) { + throw new UnauthorizedException( + authentication.getActor().toUrnStr() + " is unauthorized to " + READ + " entities."); + } + + return ResponseEntity.ok( + buildScrollResult( + opContext, + result.getEntities(), + entityAspectsBody.getAspects(), + withSystemMetadata, + result.getScrollId(), + entityAspectsBody.getAspects() != null)); } @Override @@ -118,10 +222,13 @@ public GenericEntityScrollResultV3 buildScrollResult( SearchEntityArray searchEntities, Set aspectNames, boolean withSystemMetadata, - @Nullable String scrollId) + @Nullable String scrollId, + boolean expandEmpty) throws URISyntaxException { return GenericEntityScrollResultV3.builder() - .entities(toRecordTemplates(opContext, searchEntities, aspectNames, withSystemMetadata)) + .entities( + toRecordTemplates( + opContext, searchEntities, aspectNames, withSystemMetadata, expandEmpty)) .scrollId(scrollId) .build(); } @@ -129,15 +236,16 @@ public GenericEntityScrollResultV3 buildScrollResult( @Override protected List buildEntityVersionedAspectList( @Nonnull OperationContext opContext, + Collection requestedUrns, LinkedHashMap> urnAspectVersions, - boolean withSystemMetadata) + boolean withSystemMetadata, + boolean expandEmpty) throws URISyntaxException { - if (urnAspectVersions.isEmpty()) { - return List.of(); - } else { + + if (!urnAspectVersions.isEmpty()) { Map> aspects = entityService.getEnvelopedVersionedAspects( - opContext, resolveAspectNames(urnAspectVersions, 0L), false); + opContext, resolveAspectNames(urnAspectVersions, 0L, expandEmpty), false); return urnAspectVersions.keySet().stream() .filter(urn -> aspects.containsKey(urn) && !aspects.get(urn).isEmpty()) @@ -147,7 +255,13 @@ protected List buildEntityVersionedAspectList( .build( objectMapper, u, toAspectItemMap(u, aspects.get(u), withSystemMetadata))) .collect(Collectors.toList()); + } else if (!expandEmpty) { + return requestedUrns.stream() + .map(u -> GenericEntityV3.builder().build(objectMapper, u, Collections.emptyMap())) + .collect(Collectors.toList()); } + + return List.of(); } private Map toAspectItemMap( @@ -158,7 +272,7 @@ private Map toAspectItemMap( Map.entry( a.getName(), AspectItem.builder() - .aspect(toRecordTemplate(lookupAspectSpec(urn, a.getName()), a)) + .aspect(toRecordTemplate(lookupAspectSpec(urn, a.getName()).get(), a)) .systemMetadata(withSystemMetadata ? a.getSystemMetadata() : null) .auditStamp(withSystemMetadata ? a.getCreated() : null) .build())) @@ -218,13 +332,15 @@ private List toRecordTemplates( @Nonnull OperationContext opContext, SearchEntityArray searchEntities, Set aspectNames, - boolean withSystemMetadata) + boolean withSystemMetadata, + boolean expandEmpty) throws URISyntaxException { return buildEntityList( opContext, searchEntities.stream().map(SearchEntity::getEntity).collect(Collectors.toList()), aspectNames, - withSystemMetadata); + withSystemMetadata, + expandEmpty); } private LinkedHashMap> toEntityVersionRequest( @@ -250,7 +366,7 @@ private LinkedHashMap> toEntityVersionRequest( continue; } - AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()); + AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()).orElse(null); if (aspectSpec != null) { @@ -307,7 +423,7 @@ protected AspectsBatch toMCPBatch( continue; } - AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()); + AspectSpec aspectSpec = lookupAspectSpec(entityUrn, aspect.getKey()).orElse(null); if (aspectSpec != null) { From 43c185df01e2441a2d884b8682680af59a0c5832 Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Fri, 11 Oct 2024 11:27:11 +0900 Subject: [PATCH 06/17] fix: display demo form modal on mobile (#11581) --- .../pages/cloud/DemoForm/styles.module.scss | 4 +- .../src/pages/cloud/DemoFormModal/index.jsx | 17 +++++++ .../cloud/DemoFormModal/styles.module.scss | 45 +++++++++++++++++ docs-website/src/pages/cloud/Hero/index.js | 48 +++++++++++++------ .../src/pages/cloud/Hero/styles.module.scss | 6 ++- docs-website/src/pages/cloud/index.js | 31 ++++++++---- .../src/pages/cloud/styles.module.scss | 13 ++++- 7 files changed, 135 insertions(+), 29 deletions(-) create mode 100644 docs-website/src/pages/cloud/DemoFormModal/index.jsx create mode 100644 docs-website/src/pages/cloud/DemoFormModal/styles.module.scss diff --git a/docs-website/src/pages/cloud/DemoForm/styles.module.scss b/docs-website/src/pages/cloud/DemoForm/styles.module.scss index 4157a228ae739..c56dd1d598605 100644 --- a/docs-website/src/pages/cloud/DemoForm/styles.module.scss +++ b/docs-website/src/pages/cloud/DemoForm/styles.module.scss @@ -46,9 +46,7 @@ .bookButton { display: block; } - .formContainer { - display: none; - } + .productTourButton { text-align: center!important; } diff --git a/docs-website/src/pages/cloud/DemoFormModal/index.jsx b/docs-website/src/pages/cloud/DemoFormModal/index.jsx new file mode 100644 index 0000000000000..7c890950bda97 --- /dev/null +++ b/docs-website/src/pages/cloud/DemoFormModal/index.jsx @@ -0,0 +1,17 @@ +import React, { useEffect } from 'react'; +import styles from "./styles.module.scss"; +import DemoForm from '../DemoForm'; + +const DemoFormModal = ({ formId, handleCloseModal }) => { + + return ( +
+
+ + +
+
+ ); +}; + +export default DemoFormModal; diff --git a/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss b/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss new file mode 100644 index 0000000000000..44461567e578b --- /dev/null +++ b/docs-website/src/pages/cloud/DemoFormModal/styles.module.scss @@ -0,0 +1,45 @@ + +.modal { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0, 0, 0, 0.5); /* Modal background */ + display: flex; + justify-content: center; + align-items: center; + z-index: 1000; + overflow-y: scroll; + + .modalContent { + width: 90%; + max-width: 540px; + background: #fff; + border-radius: 16px; + position: relative; + box-shadow: 0px 0px 20px rgba(0, 0, 0, 0.1); + animation: modalShow 0.3s ease-in-out; + + .closeButton { + position: absolute; + top: 1rem; + right: 1rem; + background: none; + border: none; + font-size: 1.5rem; + cursor: pointer; + } + } +} + +@keyframes modalShow { + from { + opacity: 0; + transform: translateY(-20px); + } + to { + opacity: 1; + transform: translateY(0); + } +} diff --git a/docs-website/src/pages/cloud/Hero/index.js b/docs-website/src/pages/cloud/Hero/index.js index eeb982544ddca..8e4cb2145c514 100644 --- a/docs-website/src/pages/cloud/Hero/index.js +++ b/docs-website/src/pages/cloud/Hero/index.js @@ -1,38 +1,56 @@ -import React, { useEffect } from 'react'; +import React, { useState } from 'react'; import clsx from "clsx"; import Link from "@docusaurus/Link"; import styles from "./styles.module.scss"; import ScrollingCustomers from '../CompanyLogos'; import DemoForm from '../DemoForm'; +import DemoFormModal from '../DemoFormModal'; const Hero = () => { + const [isModalOpen, setIsModalOpen] = useState(false); + + const handleOpenModal = () => setIsModalOpen(true); + const handleCloseModal = () => setIsModalOpen(false); + return (
-

DataHub Cloud

-
+

DataHub Cloud

+
Experience the premium version of DataHub -
- with Observability and Governance built-in. -
+
+ with Observability and Governance built-in.
- - Book Demo - - - Live Product Tour → - - +
+ + + + + Live Product Tour → + +
-
- +
+
+ + {isModalOpen && ( + + )}
); }; diff --git a/docs-website/src/pages/cloud/Hero/styles.module.scss b/docs-website/src/pages/cloud/Hero/styles.module.scss index 5da028d017155..c073b8590d214 100644 --- a/docs-website/src/pages/cloud/Hero/styles.module.scss +++ b/docs-website/src/pages/cloud/Hero/styles.module.scss @@ -64,6 +64,10 @@ .productTourButton { text-align: center!important; } + + .hideOnMobile{ + display: none; + } } @media screen and (min-width: 1000px){ @@ -71,4 +75,4 @@ padding-left: 0!important; margin-left: 0!important; } -} \ No newline at end of file +} diff --git a/docs-website/src/pages/cloud/index.js b/docs-website/src/pages/cloud/index.js index 99768ca721078..4d323450e2906 100644 --- a/docs-website/src/pages/cloud/index.js +++ b/docs-website/src/pages/cloud/index.js @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useState } from "react"; import Layout from "@theme/Layout"; import Link from "@docusaurus/Link"; import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; @@ -10,11 +10,16 @@ import UnifiedTabs from "./UnifiedTabs"; import FeatureCards from "./FeatureCards"; import Hero from "./Hero"; import DemoForm from "./DemoForm"; +import DemoFormModal from "./DemoFormModal"; function Home() { const context = useDocusaurusContext(); const { siteConfig = {} } = context; + const [isModalOpen, setIsModalOpen] = useState(false); + + const handleOpenModal = () => setIsModalOpen(true); + const handleCloseModal = () => setIsModalOpen(false); if (siteConfig.customFields.isSaas) { window.location.replace("/docs"); } @@ -28,7 +33,7 @@ function Home() {
- +
@@ -41,25 +46,33 @@ function Home() {
-
-

Start your free trial
today.

-
Unify Discovery, Observability and Governance
for data and AI.
+
+

Start your free trial
today.

+
+ Unify Discovery, Observability and Governance
for data and AI. +
- + Live Product Tour →
-
- +
+
+ {isModalOpen && ( + + )} ) : null; } diff --git a/docs-website/src/pages/cloud/styles.module.scss b/docs-website/src/pages/cloud/styles.module.scss index 0590b8baf12c8..1a873df9a8118 100644 --- a/docs-website/src/pages/cloud/styles.module.scss +++ b/docs-website/src/pages/cloud/styles.module.scss @@ -86,11 +86,22 @@ @media screen and (max-width: 999px) { .bookButton, .productTourButton { display: block; + width: 100%; } .productTourButton { text-align: center!important; } + + + .hideOnMobile { + display: none; + } + + .formContainer { + display: none; + } + } @media screen and (min-width: 1000px){ @@ -98,4 +109,4 @@ padding-left: 0!important; margin-left: 0!important; } -} \ No newline at end of file +} From d0d09a09f8b70054c68f26373663d0424bd81413 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:55:27 +0530 Subject: [PATCH 07/17] fix(ingest): ignore irrelevant urns from % change computation (#11583) --- .../source/state/entity_removal_state.py | 26 ++++++++++++++++++- .../state/stale_entity_removal_handler.py | 10 +++---- .../test_stale_entity_removal_handler.py | 18 +++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py index f011aa7bdd19e..318395d4e66b2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py @@ -8,6 +8,11 @@ from datahub.utilities.dedup_list import deduplicate_list from datahub.utilities.urns.urn import guess_entity_type +STATEFUL_INGESTION_IGNORED_ENTITY_TYPES = { + "dataProcessInstance", + "query", +} + def pydantic_state_migrator(mapping: Dict[str, str]) -> classmethod: # mapping would be something like: @@ -127,8 +132,11 @@ def get_percent_entities_changed( :param old_checkpoint_state: the old checkpoint state to compute the relative change percent against. :return: (1-|intersection(self, old_checkpoint_state)| / |old_checkpoint_state|) * 100.0 """ + + old_urns_filtered = filter_ignored_entity_types(old_checkpoint_state.urns) + return compute_percent_entities_changed( - new_entities=self.urns, old_entities=old_checkpoint_state.urns + new_entities=self.urns, old_entities=old_urns_filtered ) def urn_count(self) -> int: @@ -153,3 +161,19 @@ def _get_entity_overlap_and_cardinalities( new_set = set(new_entities) old_set = set(old_entities) return len(new_set.intersection(old_set)), len(old_set), len(new_set) + + +def filter_ignored_entity_types(urns: List[str]) -> List[str]: + # We previously stored ignored entity urns (e.g.dataProcessInstance) in state. + # For smoother transition from old checkpoint state, without requiring explicit + # setting of `fail_safe_threshold` due to removal of irrelevant urns from new state, + # here, we would ignore irrelevant urns from percentage entities changed computation + # This special handling can be removed after few months. + return [ + urn + for urn in urns + if not any( + urn.startswith(f"urn:li:{entityType}") + for entityType in STATEFUL_INGESTION_IGNORED_ENTITY_TYPES + ) + ] diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py index 9d77e13a0f3c2..d4fcbf09924e9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py @@ -11,7 +11,10 @@ from datahub.ingestion.api.ingestion_job_checkpointing_provider_base import JobId from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.state.checkpoint import Checkpoint -from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState +from datahub.ingestion.source.state.entity_removal_state import ( + STATEFUL_INGESTION_IGNORED_ENTITY_TYPES, + GenericCheckpointState, +) from datahub.ingestion.source.state.stateful_ingestion_base import ( StatefulIngestionConfig, StatefulIngestionConfigBase, @@ -27,11 +30,6 @@ logger: logging.Logger = logging.getLogger(__name__) -STATEFUL_INGESTION_IGNORED_ENTITY_TYPES = { - "dataProcessInstance", - "query", -} - class StatefulStaleMetadataRemovalConfig(StatefulIngestionConfig): """ diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py index 62a42b954daf8..be2d8bac12e38 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_stale_entity_removal_handler.py @@ -4,6 +4,7 @@ from datahub.ingestion.source.state.entity_removal_state import ( compute_percent_entities_changed, + filter_ignored_entity_types, ) EntList = List[str] @@ -46,3 +47,20 @@ def test_change_percent( new_entities=new_entities, old_entities=old_entities ) assert actual_percent_change == expected_percent_change + + +def test_filter_ignored_entity_types(): + + assert filter_ignored_entity_types( + [ + "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)", + "urn:li:dataProcessInstance:478810e859f870a54f72c681f41af619", + "urn:li:query:query1", + ] + ) == [ + "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset2,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,dummy_dataset3,PROD)", + ] From 59a2f708580459433fe2e61cb96433f04a79c564 Mon Sep 17 00:00:00 2001 From: sid-acryl <155424659+sid-acryl@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:24:50 +0530 Subject: [PATCH 08/17] feat(ingest/powerbi): fix subTypes and add workspace_type_filter (#11523) Co-authored-by: Harshal Sheth --- .../docs/sources/powerbi/powerbi_pre.md | 4 +- .../ingestion/source/common/subtypes.py | 5 +- .../ingestion/source/powerbi/config.py | 27 +- .../ingestion/source/powerbi/powerbi.py | 80 +- .../powerbi/rest_api_wrapper/data_classes.py | 9 +- .../powerbi/rest_api_wrapper/data_resolver.py | 105 +- .../powerbi/rest_api_wrapper/powerbi_api.py | 73 +- .../powerbi/golden_test_admin_only.json | 73 +- .../integration/powerbi/golden_test_cll.json | 30 +- .../powerbi/golden_test_container.json | 1480 ++++++++++++++--- .../golden_test_disabled_ownership.json | 27 +- .../powerbi/golden_test_endorsement.json | 27 +- .../golden_test_independent_datasets.json | 15 +- .../powerbi/golden_test_ingest.json | 27 +- .../golden_test_ingest_patch_disabled.json | 27 +- .../powerbi/golden_test_lineage.json | 27 +- .../golden_test_lower_case_urn_ingest.json | 27 +- ..._config_and_modified_since_admin_only.json | 125 +- .../powerbi/golden_test_personal_ingest.json | 329 ++++ .../golden_test_platform_instance_ingest.json | 27 +- .../powerbi/golden_test_profiling.json | 7 +- .../powerbi/golden_test_report.json | 725 +++++++- .../golden_test_scan_all_workspaces.json | 27 +- ...lden_test_server_to_platform_instance.json | 27 +- .../powerbi/test_admin_only_api.py | 15 +- .../tests/integration/powerbi/test_powerbi.py | 208 ++- .../integration/powerbi/test_profiling.py | 12 +- .../powerbi/test_stateful_ingestion.py | 21 +- 28 files changed, 2870 insertions(+), 716 deletions(-) create mode 100644 metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json diff --git a/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md b/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md index b581e5fc8f70d..f2745d5e77f49 100644 --- a/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md +++ b/metadata-ingestion/docs/sources/powerbi/powerbi_pre.md @@ -18,9 +18,11 @@ | `Report.webUrl` | `Chart.externalUrl` | | `Workspace` | `Container` | | `Report` | `Dashboard` | +| `PaginatedReport` | `Dashboard` | | `Page` | `Chart` | -If Tile is created from report then Chart.externalUrl is set to Report.webUrl. +- If `Tile` is created from report then `Chart.externalUrl` is set to Report.webUrl. +- The `Page` is unavailable for PowerBI PaginatedReport. ## Lineage diff --git a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py index 86c1c8db11b05..b6aa8c1f5f1f1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/common/subtypes.py @@ -14,7 +14,6 @@ class DatasetSubTypes(StrEnum): ELASTIC_DATASTREAM = "Datastream" SALESFORCE_CUSTOM_OBJECT = "Custom Object" SALESFORCE_STANDARD_OBJECT = "Object" - POWERBI_DATASET_TABLE = "PowerBI Dataset Table" QLIK_DATASET = "Qlik Dataset" BIGQUERY_TABLE_SNAPSHOT = "Bigquery Table Snapshot" SHARDED_TABLE = "Sharded Table" @@ -48,8 +47,8 @@ class BIContainerSubTypes(StrEnum): LOOKML_PROJECT = "LookML Project" LOOKML_MODEL = "LookML Model" TABLEAU_WORKBOOK = "Workbook" - POWERBI_WORKSPACE = "Workspace" - POWERBI_DATASET = "PowerBI Dataset" + POWERBI_DATASET = "Semantic Model" + POWERBI_DATASET_TABLE = "Table" QLIK_SPACE = "Qlik Space" QLIK_APP = "Qlik App" SIGMA_WORKSPACE = "Sigma Workspace" diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py index 967dd5d81112d..522639a160781 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py @@ -1,7 +1,7 @@ import logging from dataclasses import dataclass, field as dataclass_field from enum import Enum -from typing import Dict, List, Optional, Union +from typing import Dict, List, Literal, Optional, Union import pydantic from pydantic import validator @@ -47,6 +47,7 @@ class Constant: WORKSPACE_ID = "workspaceId" DASHBOARD_ID = "powerbi.linkedin.com/dashboards/{}" DATASET_EXECUTE_QUERIES = "DATASET_EXECUTE_QUERIES_POST" + GET_WORKSPACE_APP = "GET_WORKSPACE_APP" DATASET_ID = "datasetId" REPORT_ID = "reportId" SCAN_ID = "ScanId" @@ -118,6 +119,15 @@ class Constant: CHART_COUNT = "chartCount" WORKSPACE_NAME = "workspaceName" DATASET_WEB_URL = "datasetWebUrl" + TYPE = "type" + REPORT_TYPE = "reportType" + LAST_UPDATE = "lastUpdate" + APP_ID = "appId" + REPORTS = "reports" + ORIGINAL_REPORT_OBJECT_ID = "originalReportObjectId" + APP_SUB_TYPE = "App" + STATE = "state" + ACTIVE = "Active" @dataclass @@ -273,7 +283,8 @@ class PowerBiDashboardSourceConfig( # PowerBi workspace identifier workspace_id_pattern: AllowDenyPattern = pydantic.Field( default=AllowDenyPattern.allow_all(), - description="Regex patterns to filter PowerBI workspaces in ingestion", + description="Regex patterns to filter PowerBI workspaces in ingestion." + " Note: This field works in conjunction with 'workspace_type_filter' and both must be considered when filtering workspaces.", ) # Dataset type mapping PowerBI support many type of data-sources. Here user need to define what type of PowerBI @@ -340,7 +351,7 @@ class PowerBiDashboardSourceConfig( ) modified_since: Optional[str] = pydantic.Field( default=None, - description="Get only recently modified workspaces based on modified_since datetime '2023-02-10T00:00:00.0000000Z', excludePersonalWorkspaces and excludeInActiveWorkspaces limit to last 30 days", + description="Get only recently modified workspaces based on modified_since datetime '2023-02-10T00:00:00.0000000Z', excludeInActiveWorkspaces limit to last 30 days", ) extract_dashboards: bool = pydantic.Field( default=True, @@ -445,6 +456,16 @@ class PowerBiDashboardSourceConfig( description="Patch dashboard metadata", ) + workspace_type_filter: List[ + Literal[ + "Workspace", "PersonalGroup", "Personal", "AdminWorkspace", "AdminInsights" + ] + ] = pydantic.Field( + default=["Workspace"], + description="Ingest the metadata of the workspace where the workspace type corresponds to the specified workspace_type_filter." + " Note: This field works in conjunction with 'workspace_id_pattern'. Both must be matched for a workspace to be processed.", + ) + @root_validator(skip_on_failure=True) def validate_extract_column_level_lineage(cls, values: Dict) -> Dict: flags = [ diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py index 065bbac9e9645..f5c0aedb329cd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py @@ -34,7 +34,6 @@ from datahub.ingestion.source.common.subtypes import ( BIAssetSubTypes, BIContainerSubTypes, - DatasetSubTypes, ) from datahub.ingestion.source.powerbi.config import ( Constant, @@ -142,9 +141,7 @@ def assets_urn_to_lowercase(self, value): def new_mcp( self, - entity_type, entity_urn, - aspect_name, aspect, change_type=ChangeTypeClass.UPSERT, ): @@ -152,10 +149,8 @@ def new_mcp( Create MCP """ return MetadataChangeProposalWrapper( - entityType=entity_type, changeType=change_type, entityUrn=entity_urn, - aspectName=aspect_name, aspect=aspect, ) @@ -176,9 +171,7 @@ def extract_dataset_schema( ) -> List[MetadataChangeProposalWrapper]: schema_metadata = self.to_datahub_schema(table) schema_mcp = self.new_mcp( - entity_type=Constant.DATASET, entity_urn=ds_urn, - aspect_name=Constant.SCHEMA_METADATA, aspect=schema_metadata, ) return [schema_mcp] @@ -409,9 +402,7 @@ def to_datahub_dataset( viewLanguage="m_query", ) view_prop_mcp = self.new_mcp( - entity_type=Constant.DATASET, entity_urn=ds_urn, - aspect_name=Constant.VIEW_PROPERTIES, aspect=view_properties, ) dataset_mcps.extend([view_prop_mcp]) @@ -425,30 +416,23 @@ def to_datahub_dataset( ) info_mcp = self.new_mcp( - entity_type=Constant.DATASET, entity_urn=ds_urn, - aspect_name=Constant.DATASET_PROPERTIES, aspect=ds_properties, ) # Remove status mcp status_mcp = self.new_mcp( - entity_type=Constant.DATASET, entity_urn=ds_urn, - aspect_name=Constant.STATUS, aspect=StatusClass(removed=False), ) if self.__config.extract_dataset_schema: dataset_mcps.extend(self.extract_dataset_schema(table, ds_urn)) subtype_mcp = self.new_mcp( - entity_type=Constant.DATASET, entity_urn=ds_urn, - aspect_name=Constant.SUBTYPES, aspect=SubTypesClass( typeNames=[ - DatasetSubTypes.POWERBI_DATASET_TABLE, - DatasetSubTypes.VIEW, + BIContainerSubTypes.POWERBI_DATASET_TABLE, ] ), ) @@ -464,9 +448,7 @@ def to_datahub_dataset( # Dashboard owner MCP ownership = OwnershipClass(owners=[owner_class]) owner_mcp = self.new_mcp( - entity_type=Constant.DATASET, entity_urn=ds_urn, - aspect_name=Constant.OWNERSHIP, aspect=ownership, ) dataset_mcps.extend([owner_mcp]) @@ -606,17 +588,13 @@ def tile_custom_properties(tile: powerbi_data_classes.Tile) -> dict: ) info_mcp = self.new_mcp( - entity_type=Constant.CHART, entity_urn=chart_urn, - aspect_name=Constant.CHART_INFO, aspect=chart_info_instance, ) # removed status mcp status_mcp = self.new_mcp( - entity_type=Constant.CHART, entity_urn=chart_urn, - aspect_name=Constant.STATUS, aspect=StatusClass(removed=False), ) @@ -633,18 +611,14 @@ def tile_custom_properties(tile: powerbi_data_classes.Tile) -> dict: # Explicitly emitting this aspect isn't necessary, but we do it here to ensure that # the old, bad data gets overwritten. chart_key_mcp = self.new_mcp( - entity_type=Constant.CHART, entity_urn=chart_urn, - aspect_name=Constant.CHART_KEY, aspect=ChartUrn.from_string(chart_urn).to_key_aspect(), ) # Browse path browse_path = BrowsePathsClass(paths=[f"/powerbi/{workspace.name}"]) browse_path_mcp = self.new_mcp( - entity_type=Constant.CHART, entity_urn=chart_urn, - aspect_name=Constant.BROWSERPATH, aspect=browse_path, ) result_mcps = [ @@ -710,17 +684,13 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict: ) info_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.DASHBOARD_INFO, aspect=dashboard_info_cls, ) # removed status mcp removed_status_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.STATUS, aspect=StatusClass(removed=False), ) @@ -732,9 +702,7 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict: # Dashboard key dashboard_key_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.DASHBOARD_KEY, aspect=dashboard_key_cls, ) @@ -750,9 +718,7 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict: # Dashboard owner MCP ownership = OwnershipClass(owners=owners) owner_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.OWNERSHIP, aspect=ownership, ) @@ -761,9 +727,7 @@ def chart_custom_properties(dashboard: powerbi_data_classes.Dashboard) -> dict: paths=[f"/{Constant.PLATFORM_NAME}/{dashboard.workspace_name}"] ) browse_path_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.BROWSERPATH, aspect=browse_path, ) @@ -827,7 +791,7 @@ def generate_container_for_workspace( container_work_units = gen_containers( container_key=self.workspace_key, name=workspace.name, - sub_types=[BIContainerSubTypes.POWERBI_WORKSPACE], + sub_types=[workspace.type], ) return container_work_units @@ -858,9 +822,7 @@ def append_tag_mcp( ) -> None: if self.__config.extract_endorsements_to_tags and tags: tags_mcp = self.new_mcp( - entity_type=entity_type, entity_urn=entity_urn, - aspect_name=Constant.GLOBAL_TAGS, aspect=self.transform_tags(tags), ) list_of_mcps.append(tags_mcp) @@ -883,9 +845,7 @@ def to_datahub_user( user_key = CorpUserKeyClass(username=user.id) user_key_mcp = self.new_mcp( - entity_type=Constant.CORP_USER, entity_urn=user_urn, - aspect_name=Constant.CORP_USER_KEY, aspect=user_key, ) @@ -1028,17 +988,13 @@ def to_chart_mcps( ) info_mcp = self.new_mcp( - entity_type=Constant.CHART, entity_urn=chart_urn, - aspect_name=Constant.CHART_INFO, aspect=chart_info_instance, ) # removed status mcp status_mcp = self.new_mcp( - entity_type=Constant.CHART, entity_urn=chart_urn, - aspect_name=Constant.STATUS, aspect=StatusClass(removed=False), ) # Subtype mcp @@ -1052,9 +1008,7 @@ def to_chart_mcps( # Browse path browse_path = BrowsePathsClass(paths=[f"/powerbi/{workspace.name}"]) browse_path_mcp = self.new_mcp( - entity_type=Constant.CHART, entity_urn=chart_urn, - aspect_name=Constant.BROWSERPATH, aspect=browse_path, ) list_of_mcps = [info_mcp, status_mcp, subtype_mcp, browse_path_mcp] @@ -1105,17 +1059,13 @@ def report_to_dashboard( ) info_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.DASHBOARD_INFO, aspect=dashboard_info_cls, ) # removed status mcp removed_status_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.STATUS, aspect=StatusClass(removed=False), ) @@ -1127,9 +1077,7 @@ def report_to_dashboard( # Dashboard key dashboard_key_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.DASHBOARD_KEY, aspect=dashboard_key_cls, ) # Report Ownership @@ -1144,9 +1092,7 @@ def report_to_dashboard( # Report owner MCP ownership = OwnershipClass(owners=owners) owner_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.OWNERSHIP, aspect=ownership, ) @@ -1155,17 +1101,13 @@ def report_to_dashboard( paths=[f"/{Constant.PLATFORM_NAME}/{workspace.name}"] ) browse_path_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=Constant.BROWSERPATH, aspect=browse_path, ) sub_type_mcp = self.new_mcp( - entity_type=Constant.DASHBOARD, entity_urn=dashboard_urn, - aspect_name=SubTypesClass.ASPECT_NAME, - aspect=SubTypesClass(typeNames=[Constant.REPORT_TYPE_NAME]), + aspect=SubTypesClass(typeNames=[report.type.value]), ) list_of_mcps = [ @@ -1203,7 +1145,7 @@ def report_to_datahub_work_units( logger.debug(f"Converting report={report.name} to datahub dashboard") # Convert user to CorpUser user_mcps = self.to_datahub_users(report.users) - # Convert pages to charts. A report has single dataset and same dataset used in pages to create visualization + # Convert pages to charts. A report has a single dataset and the same dataset used in pages to create visualization ds_mcps = self.to_datahub_dataset(report.dataset, workspace) chart_mcps = self.pages_to_chart(report.pages, workspace, ds_mcps) @@ -1267,7 +1209,10 @@ def __init__(self, config: PowerBiDashboardSourceConfig, ctx: PipelineContext): self.source_config ) try: - self.powerbi_client = PowerBiAPI(self.source_config) + self.powerbi_client = PowerBiAPI( + config=self.source_config, + reporter=self.reporter, + ) except Exception as e: logger.warning(e) exit( @@ -1288,7 +1233,10 @@ def __init__(self, config: PowerBiDashboardSourceConfig, ctx: PipelineContext): def test_connection(config_dict: dict) -> TestConnectionReport: test_report = TestConnectionReport() try: - PowerBiAPI(PowerBiDashboardSourceConfig.parse_obj_allow_extras(config_dict)) + PowerBiAPI( + PowerBiDashboardSourceConfig.parse_obj_allow_extras(config_dict), + PowerBiDashboardSourceReport(), + ) test_report.basic_connectivity = CapabilityReport(capable=True) except Exception as e: test_report.basic_connectivity = CapabilityReport( @@ -1308,6 +1256,7 @@ def get_allowed_workspaces(self) -> List[powerbi_data_classes.Workspace]: workspace for workspace in all_workspaces if self.source_config.workspace_id_pattern.allowed(workspace.id) + and workspace.type in self.source_config.workspace_type_filter ] logger.info(f"Number of workspaces = {len(all_workspaces)}") @@ -1366,8 +1315,9 @@ def get_workspace_workunit( ) for workunit in workspace_workunits: - # Return workunit to Datahub Ingestion framework + # Return workunit to a Datahub Ingestion framework yield workunit + for dashboard in workspace.dashboards: try: # Fetch PowerBi users for dashboards diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py index 5106b9817d351..fb0959ac604c4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py @@ -41,6 +41,7 @@ class DatasetKey(ContainerKey): class Workspace: id: str name: str + type: str # This is used as a subtype of the Container entity. dashboards: List["Dashboard"] reports: List["Report"] datasets: Dict[str, "PowerBIDataset"] @@ -211,10 +212,16 @@ def __hash__(self): return hash(self.__members()) +class ReportType(Enum): + PaginatedReport = "PaginatedReport" + PowerBIReport = "Report" + + @dataclass class Report: id: str name: str + type: ReportType webUrl: Optional[str] embedUrl: str description: str @@ -259,7 +266,7 @@ class Dashboard: tiles: List["Tile"] users: List["User"] tags: List[str] - webUrl: Optional[str] = None + webUrl: Optional[str] def get_urn_part(self): return f"dashboards.{self.id}" diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py index b190cf065b6e3..d89b9662d12ed 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py @@ -1,9 +1,8 @@ import logging -import math from abc import ABC, abstractmethod from datetime import datetime, timedelta from time import sleep -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Iterator, List, Optional, Union import msal import requests @@ -21,6 +20,7 @@ Page, PowerBIDataset, Report, + ReportType, Table, Tile, User, @@ -57,7 +57,8 @@ def is_http_failure(response: Response, message: str) -> bool: class DataResolverBase(ABC): SCOPE: str = "https://analysis.windows.net/powerbi/api/.default" - BASE_URL: str = "https://api.powerbi.com/v1.0/myorg/groups" + MY_ORG_URL = "https://api.powerbi.com/v1.0/myorg" + BASE_URL: str = f"{MY_ORG_URL}/groups" ADMIN_BASE_URL: str = "https://api.powerbi.com/v1.0/myorg/admin" AUTHORITY: str = "https://login.microsoftonline.com/" TOP: int = 1000 @@ -222,49 +223,27 @@ def get_dashboards(self, workspace: Workspace) -> List[Dashboard]: tags=[], ) for instance in dashboards_dict - if instance is not None + if ( + instance is not None + and Constant.APP_ID + not in instance # As we add dashboards to the App, Power BI starts + # providing duplicate dashboard information, + # where the duplicate includes an AppId, while the original dashboard does not. + ) ] return dashboards - def get_groups(self) -> List[dict]: + def get_groups(self, filter_: Dict) -> List[dict]: group_endpoint = self.get_groups_endpoint() - params: dict = {"$top": self.TOP, "$skip": 0, "$filter": "type eq 'Workspace'"} - - def fetch_page(page_number: int) -> dict: - params["$skip"] = self.TOP * page_number - logger.debug(f"Query parameters = {params}") - response = self._request_session.get( - group_endpoint, - headers=self.get_authorization_header(), - params=params, - ) - response.raise_for_status() - return response.json() - # Hit PowerBi - logger.debug(f"Request to groups endpoint URL={group_endpoint}") - zeroth_page = fetch_page(0) - logger.debug(f"Page 0 = {zeroth_page}") - if zeroth_page.get(Constant.ODATA_COUNT) is None: - logger.warning( - "@odata.count field is not present in response. Unable to fetch workspaces." - ) - return [] + output: List[dict] = [] - number_of_items = zeroth_page[Constant.ODATA_COUNT] - number_of_pages = math.ceil(number_of_items / self.TOP) - output: List[dict] = zeroth_page[Constant.VALUE] - for page in range( - 1, number_of_pages - ): # start from 1 as 0th index already fetched - page_response = fetch_page(page) - if len(page_response[Constant.VALUE]) == 0: - break - - logger.debug(f"Page {page} = {zeroth_page}") - - output.extend(page_response[Constant.VALUE]) + for page in self.itr_pages( + endpoint=group_endpoint, + parameter_override=filter_, + ): + output.extend(page) return output @@ -286,13 +265,14 @@ def fetch_reports(): ) response.raise_for_status() response_dict = response.json() - logger.debug(f"Request response = {response_dict}") + logger.debug(f"Report Request response = {response_dict}") return response_dict.get(Constant.VALUE, []) reports: List[Report] = [ Report( id=raw_instance.get(Constant.ID), name=raw_instance.get(Constant.NAME), + type=ReportType[raw_instance.get(Constant.REPORT_TYPE)], webUrl=raw_instance.get(Constant.WEB_URL), embedUrl=raw_instance.get(Constant.EMBED_URL), description=raw_instance.get(Constant.DESCRIPTION, ""), @@ -304,6 +284,11 @@ def fetch_reports(): dataset=workspace.datasets.get(raw_instance.get(Constant.DATASET_ID)), ) for raw_instance in fetch_reports() + if Constant.APP_ID + not in raw_instance # As we add reports to the App, Power BI starts providing + # duplicate report information, + # where the duplicate includes an AppId, + # while the original report does not. ] return reports @@ -395,6 +380,40 @@ def new_dataset_or_report(tile_instance: Any) -> dict: return tiles + def itr_pages( + self, + endpoint: str, + parameter_override: Dict = {}, + ) -> Iterator[List[Dict]]: + params: dict = { + "$skip": 0, + "$top": self.TOP, + **parameter_override, + } + + page_number: int = 0 + + while True: + params["$skip"] = self.TOP * page_number + response = self._request_session.get( + endpoint, + headers=self.get_authorization_header(), + params=params, + ) + + response.raise_for_status() + + assert ( + Constant.VALUE in response.json() + ), "'value' key is not present in paginated response" + + if not response.json()[Constant.VALUE]: # if it is an empty list then break + break + + yield response.json()[Constant.VALUE] + + page_number += 1 + class RegularAPIResolver(DataResolverBase): # Regular access endpoints @@ -407,6 +426,7 @@ class RegularAPIResolver(DataResolverBase): Constant.REPORT_LIST: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports", Constant.PAGE_BY_REPORT: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/reports/{REPORT_ID}/pages", Constant.DATASET_EXECUTE_QUERIES: "{POWERBI_BASE_URL}/{WORKSPACE_ID}/datasets/{DATASET_ID}/executeQueries", + Constant.GET_WORKSPACE_APP: "{MY_ORG_URL}/apps/{APP_ID}", } def get_dataset( @@ -676,6 +696,7 @@ class AdminAPIResolver(DataResolverBase): Constant.ENTITY_USER_LIST: "{POWERBI_ADMIN_BASE_URL}/{ENTITY}/{ENTITY_ID}/users", Constant.DATASET_LIST: "{POWERBI_ADMIN_BASE_URL}/groups/{WORKSPACE_ID}/datasets", Constant.WORKSPACE_MODIFIED_LIST: "{POWERBI_ADMIN_BASE_URL}/workspaces/modified", + Constant.GET_WORKSPACE_APP: "{POWERBI_ADMIN_BASE_URL}/apps", } def create_scan_job(self, workspace_ids: List[str]) -> str: @@ -922,7 +943,7 @@ def _get_pages_by_report(self, workspace: Workspace, report_id: str) -> List[Pag def get_modified_workspaces(self, modified_since: str) -> List[str]: """ - Get list of modified workspaces + Get a list of modified workspaces """ modified_workspaces_endpoint = self.API_ENDPOINTS[ Constant.WORKSPACE_MODIFIED_LIST @@ -930,7 +951,7 @@ def get_modified_workspaces(self, modified_since: str) -> List[str]: POWERBI_ADMIN_BASE_URL=DataResolverBase.ADMIN_BASE_URL, ) parameters: Dict[str, Any] = { - "excludePersonalWorkspaces": True, + "excludePersonalWorkspaces": False, "excludeInActiveWorkspaces": True, "modifiedSince": modified_since, } diff --git a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py index a245d4c2b9a35..25e97b158d48b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/powerbi_api.py @@ -32,8 +32,13 @@ class PowerBiAPI: - def __init__(self, config: PowerBiDashboardSourceConfig) -> None: + def __init__( + self, + config: PowerBiDashboardSourceConfig, + reporter: PowerBiDashboardSourceReport, + ) -> None: self.__config: PowerBiDashboardSourceConfig = config + self.__reporter = reporter self.__regular_api_resolver = RegularAPIResolver( client_id=self.__config.client_id, @@ -182,17 +187,27 @@ def fill_tags() -> None: fill_ownership() fill_tags() - return reports def get_workspaces(self) -> List[Workspace]: + modified_workspace_ids: List[str] = [] + if self.__config.modified_since: - workspaces = self.get_modified_workspaces() - return workspaces + modified_workspace_ids = self.get_modified_workspaces() groups: List[dict] = [] + filter_: Dict[str, str] = {} try: - groups = self._get_resolver().get_groups() + if modified_workspace_ids: + id_filter: List[str] = [] + + for id_ in modified_workspace_ids: + id_filter.append(f"id eq {id_}") + + filter_["$filter"] = " or ".join(id_filter) + + groups = self._get_resolver().get_groups(filter_=filter_) + except: self.log_http_error(message="Unable to fetch list of workspaces") raise # we want this exception to bubble up @@ -201,6 +216,7 @@ def get_workspaces(self) -> List[Workspace]: Workspace( id=workspace[Constant.ID], name=workspace[Constant.NAME], + type=workspace[Constant.TYPE], datasets={}, dashboards=[], reports=[], @@ -213,34 +229,20 @@ def get_workspaces(self) -> List[Workspace]: ] return workspaces - def get_modified_workspaces(self) -> List[Workspace]: - workspaces: List[Workspace] = [] + def get_modified_workspaces(self) -> List[str]: + modified_workspace_ids: List[str] = [] if self.__config.modified_since is None: - return workspaces + return modified_workspace_ids try: modified_workspace_ids = self.__admin_api_resolver.get_modified_workspaces( self.__config.modified_since ) - workspaces = [ - Workspace( - id=workspace_id, - name="", - datasets={}, - dashboards=[], - reports=[], - report_endorsements={}, - dashboard_endorsements={}, - scan_result={}, - independent_datasets=[], - ) - for workspace_id in modified_workspace_ids - ] except: self.log_http_error(message="Unable to fetch list of modified workspaces.") - return workspaces + return modified_workspace_ids def _get_scan_result(self, workspace_ids: List[str]) -> Any: scan_id: Optional[str] = None @@ -389,9 +391,28 @@ def _fill_metadata_from_scan_result( workspaces = [] for workspace_metadata in scan_result["workspaces"]: + if ( + workspace_metadata.get(Constant.STATE) != Constant.ACTIVE + or workspace_metadata.get(Constant.TYPE) + not in self.__config.workspace_type_filter + ): + # if the state is not "Active" then in some state like Not Found, "name" attribute is not present + wrk_identifier: str = ( + workspace_metadata[Constant.NAME] + if workspace_metadata.get(Constant.NAME) + else workspace_metadata.get(Constant.ID) + ) + self.__reporter.info( + title="Skipped Workspace", + message="Workspace was skipped due to the workspace_type_filter", + context=f"workspace={wrk_identifier}", + ) + continue + cur_workspace = Workspace( - id=workspace_metadata["id"], - name=workspace_metadata["name"], + id=workspace_metadata[Constant.ID], + name=workspace_metadata[Constant.NAME], + type=workspace_metadata[Constant.TYPE], datasets={}, dashboards=[], reports=[], @@ -403,7 +424,7 @@ def _fill_metadata_from_scan_result( cur_workspace.scan_result = workspace_metadata cur_workspace.datasets = self._get_workspace_datasets(cur_workspace) - # Fetch endorsements tag if it is enabled from configuration + # Fetch endorsement tag if it is enabled from configuration if self.__config.extract_endorsements_to_tags: cur_workspace.dashboard_endorsements = self._get_dashboard_endorsements( cur_workspace.scan_result diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json index fa4bcb8abaa94..5cfa4ec80c643 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -238,8 +236,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -338,8 +335,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -446,8 +442,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -546,8 +541,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -646,8 +640,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -746,8 +739,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -846,8 +838,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1118,11 +1109,12 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "subTypes", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -1135,12 +1127,11 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartKey", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -1416,8 +1407,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1491,8 +1481,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1591,8 +1580,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1691,8 +1679,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1799,8 +1786,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1899,8 +1885,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1994,14 +1979,19 @@ "json": [ { "op": "add", - "path": "/dashboardUrl", - "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + "path": "/title", + "value": "SalesMarketing" }, { "op": "add", "path": "/description", "value": "Acryl sales marketing report" }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, { "op": "add", "path": "/lastModified", @@ -2015,11 +2005,6 @@ "actor": "urn:li:corpuser:unknown" } } - }, - { - "op": "add", - "path": "/title", - "value": "SalesMarketing" } ] }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json index 60b36897ed2e4..66ee60c2eebb3 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_cll.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -238,8 +236,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -364,8 +361,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -464,8 +460,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -596,8 +591,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -696,8 +690,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -796,8 +789,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -896,8 +888,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1465,8 +1456,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json index b43e4a6c2c1c2..e8be3aa9c0ac7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_container.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_container.json @@ -122,15 +122,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "materialized": false, - "viewLogic": "dummy", - "viewLanguage": "m_query" + "platform": "urn:li:dataPlatform:powerbi" } }, "systemMetadata": { @@ -140,19 +138,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "public issue_history", - "description": "Library dataset description", - "tags": [] + "typeNames": [ + "Semantic Model" + ] } }, "systemMetadata": { @@ -165,10 +159,10 @@ "entityType": "container", "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:powerbi" + "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" } }, "systemMetadata": { @@ -181,11 +175,14 @@ "entityType": "container", "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "PowerBI Dataset" + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } ] } }, @@ -196,18 +193,15 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "viewProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", - "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" - } - ] + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -217,13 +211,19 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "datasetProperties", "aspect": { "json": { - "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "public issue_history", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -256,8 +256,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -372,8 +371,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -488,8 +486,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -604,8 +601,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -720,8 +716,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -836,8 +831,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -952,8 +946,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1041,15 +1034,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "materialized": false, - "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", - "viewLanguage": "m_query" + "platform": "urn:li:dataPlatform:powerbi" } }, "systemMetadata": { @@ -1059,19 +1050,15 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "subTypes", "aspect": { "json": { - "customProperties": { - "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", - "name": "dbo_book_issue", - "description": "hr pbi test description", - "tags": [] + "typeNames": [ + "Semantic Model" + ] } }, "systemMetadata": { @@ -1084,10 +1071,10 @@ "entityType": "container", "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:powerbi" + "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" } }, "systemMetadata": { @@ -1100,11 +1087,14 @@ "entityType": "container", "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePathsV2", "aspect": { "json": { - "typeNames": [ - "PowerBI Dataset" + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } ] } }, @@ -1115,18 +1105,15 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "viewProperties", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", - "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" - } - ] + "materialized": false, + "viewLogic": "let\n Source = Sql.Database(\"localhost\", \"library\"),\n dbo_book_issue = Source{[Schema=\"dbo\",Item=\"book_issue\"]}[Data]\n in dbo_book_issue", + "viewLanguage": "m_query" } }, "systemMetadata": { @@ -1136,13 +1123,19 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:977b804137a1d2bf897ff1bbf440a1cc", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "datasetProperties", "aspect": { "json": { - "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + "customProperties": { + "datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed/details", + "name": "dbo_book_issue", + "description": "hr pbi test description", + "tags": [] } }, "systemMetadata": { @@ -1175,8 +1168,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1291,8 +1283,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1870,15 +1861,17 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", - "aspectName": "viewProperties", + "aspectName": "containerProperties", "aspect": { "json": { - "materialized": false, - "viewLogic": "dummy", - "viewLanguage": "m_query" + "customProperties": { + "platform": "powerbi", + "dataset": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "name": "library-dataset" } }, "systemMetadata": { @@ -1888,19 +1881,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", - "aspectName": "datasetProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" - }, - "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "public issue_history", - "description": "Library dataset description", - "tags": [] + "removed": false } }, "systemMetadata": { @@ -1910,13 +1897,13 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "removed": false + "platform": "urn:li:dataPlatform:powerbi" } }, "systemMetadata": { @@ -1926,15 +1913,14 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Semantic Model" ] } }, @@ -1944,15 +1930,31 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLogic": "dummy", "viewLanguage": "m_query" } }, @@ -1964,7 +1966,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -1973,7 +1975,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "SNOWFLAKE_TESTTABLE", + "name": "public issue_history", "description": "Library dataset description", "tags": [] } @@ -1986,7 +1988,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2002,14 +2004,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -2021,13 +2022,29 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", "viewLanguage": "m_query" } }, @@ -2039,7 +2056,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -2048,7 +2065,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "snowflake native-query", + "name": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description", "tags": [] } @@ -2061,7 +2078,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2077,14 +2094,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -2096,13 +2112,29 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "viewProperties", "aspect": { "json": { "materialized": false, - "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", "viewLanguage": "m_query" } }, @@ -2114,7 +2146,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { @@ -2123,7 +2155,7 @@ "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" }, "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", - "name": "big-query-with-parameter", + "name": "snowflake native-query", "description": "Library dataset description", "tags": [] } @@ -2136,7 +2168,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2152,14 +2184,13 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -2169,6 +2200,112 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "big-query-with-parameter", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", @@ -2233,8 +2370,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -2244,6 +2380,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", @@ -2308,8 +2460,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -2319,6 +2470,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", @@ -2383,8 +2550,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -2394,6 +2560,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", @@ -2495,6 +2677,24 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Page" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", @@ -2619,6 +2819,24 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PowerBI Page" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", @@ -2692,6 +2910,60 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/title", + "value": "SalesMarketing" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)" + }, + { + "op": "add", + "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", @@ -2774,15 +3046,13 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "container", "aspect": { "json": { - "typeNames": [ - "PowerBI Page" - ] + "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" } }, "systemMetadata": { @@ -2795,10 +3065,411 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "powerbi", + "dataset": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "name": "library-dataset" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:powerbi" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Semantic Model" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "public issue_history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "SNOWFLAKE_TESTTABLE", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "big-query-with-parameter", + "description": "Library dataset description", + "tags": [] } }, "systemMetadata": { @@ -2808,17 +3479,30 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", - "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" - } + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" ] } }, @@ -2829,8 +3513,64 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2845,8 +3585,82 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2861,13 +3675,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User4@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "subTypes", "aspect": { "json": { - "username": "User4@foo.com" + "typeNames": [ + "Table" + ] } }, "systemMetadata": { @@ -2877,8 +3693,64 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User4@foo.com", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -2893,14 +3765,14 @@ } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { "json": { "typeNames": [ - "PowerBI Page" + "Table" ] } }, @@ -2910,14 +3782,14 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User3@foo.com", +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", "changeType": "UPSERT", - "aspectName": "corpUserKey", + "aspectName": "container", "aspect": { "json": { - "username": "User3@foo.com" + "container": "urn:li:container:6ac0662f0f2fc3a9196ac505da2182b2" } }, "systemMetadata": { @@ -2927,13 +3799,15 @@ } }, { - "entityType": "corpuser", - "entityUrn": "urn:li:corpuser:users.User3@foo.com", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePaths", "aspect": { "json": { - "removed": false + "paths": [ + "/powerbi/demo-workspace" + ] } }, "systemMetadata": { @@ -2944,7 +3818,7 @@ }, { "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,reports.5b218778-e7a5-4d73-8187-f10824047715)", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", "changeType": "PATCH", "aspectName": "dashboardInfo", "aspect": { @@ -2952,27 +3826,17 @@ { "op": "add", "path": "/title", - "value": "SalesMarketing" + "value": "Printable SalesMarketing" }, { "op": "add", "path": "/description", "value": "Acryl sales marketing report" }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)", - "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection)" - }, - { - "op": "add", - "path": "/charts/urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", - "value": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)" - }, { "op": "add", "path": "/dashboardUrl", - "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715" + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163" }, { "op": "add", @@ -2997,8 +3861,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -3012,6 +3876,78 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/584cf13a-1485-41c2-a514-b1bb66fff163" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PaginatedReport" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9", + "urn": "urn:li:container:a4ed52f9abd3ff9cc34960c0c41f72e9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", @@ -3036,10 +3972,10 @@ "entityType": "container", "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [] + "removed": false } }, "systemMetadata": { @@ -3083,15 +4019,13 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", + "entityType": "container", + "entityUrn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "browsePathsV2", "aspect": { "json": { - "paths": [ - "/powerbi/second-demo-workspace" - ] + "path": [] } }, "systemMetadata": { @@ -3101,13 +4035,29 @@ } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User3@foo.com", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "corpUserKey", "aspect": { "json": { - "removed": false + "username": "User3@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User4@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User4@foo.com" } }, "systemMetadata": { @@ -3120,14 +4070,11 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "browsePaths", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", - "urn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e" - } + "paths": [ + "/powerbi/second-demo-workspace" ] } }, @@ -3195,11 +4142,10 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "dashboardKey", + "aspectName": "status", "aspect": { "json": { - "dashboardTool": "powerbi", - "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE" + "removed": false } }, "systemMetadata": { @@ -3212,10 +4158,11 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dashboardKey", "aspect": { "json": { - "container": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e" + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE" } }, "systemMetadata": { @@ -3253,5 +4200,106 @@ "runId": "powerbi-test", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-8FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e", + "urn": "urn:li:container:33c7cab6ea0e58930cd6f943d0a4111e" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User3@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User4@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json index c5414444cc35b..665f5d5a3bb41 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -213,8 +211,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -288,8 +285,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -363,8 +359,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -438,8 +433,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -513,8 +507,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -588,8 +581,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -663,8 +655,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json index e1ddbfb901bad..26476e61a0bd7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -158,8 +157,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -253,8 +251,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -348,8 +345,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -443,8 +439,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -538,8 +533,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -633,8 +627,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -728,8 +721,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -803,8 +795,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json b/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json index d204d426a38d3..0b822ad19b425 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_independent_datasets.json @@ -13,7 +13,8 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test" + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" } }, { @@ -34,7 +35,8 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test" + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" } }, { @@ -49,7 +51,8 @@ }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test" + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" } }, { @@ -60,14 +63,14 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, "systemMetadata": { "lastObserved": 1643871600000, - "runId": "powerbi-test" + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json index 6f899a7fa11b7..83f8f881835b7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -213,8 +211,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -288,8 +285,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -363,8 +359,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -438,8 +433,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -513,8 +507,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -588,8 +581,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -663,8 +655,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json index efbd9abfdb911..93a2c533d21ca 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_ingest_patch_disabled.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -213,8 +211,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -288,8 +285,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -363,8 +359,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -438,8 +433,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -513,8 +507,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -588,8 +581,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -663,8 +655,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json index 9a09cb4fec64d..eda831722cc91 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -238,8 +236,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -338,8 +335,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -413,8 +409,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -521,8 +516,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -621,8 +615,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -721,8 +714,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -821,8 +813,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json index a4eb670a4b7f9..6f502cdfc0f5b 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json @@ -167,8 +167,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -186,8 +185,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -279,8 +277,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -336,8 +333,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -409,8 +405,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -446,8 +441,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -668,8 +662,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -744,8 +737,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -819,8 +811,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json index 66e87952bf141..4393a87d1f570 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_most_config_and_modified_since_admin_only.json @@ -182,33 +182,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" - ] - } - }, - "systemMetadata": { - "lastObserved": 1643871600000, - "runId": "powerbi-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", - "changeType": "UPSERT", - "aspectName": "upstreamLineage", - "aspect": { - "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)", - "type": "TRANSFORMED" - } + "Table" ] } }, @@ -354,12 +328,21 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.dbo_book_issue,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "upstreamLineage", "aspect": { "json": { - "removed": false + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.library.dbo.book_issue,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -372,13 +355,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "PowerBI Dataset Table", - "View" - ] + "removed": false } }, "systemMetadata": { @@ -391,18 +371,11 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.COMMOPSDB.dbo.V_PS_CD_RETENTION,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -600,12 +573,21 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.ms_sql_native_table,DEV)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "upstreamLineage", "aspect": { "json": { - "removed": false + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.COMMOPSDB.dbo.V_PS_CD_RETENTION,PROD)", + "type": "TRANSFORMED" + } + ] } }, "systemMetadata": { @@ -618,13 +600,10 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "PowerBI Dataset Table", - "View" - ] + "removed": false } }, "systemMetadata": { @@ -637,18 +616,11 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "subTypes", "aspect": { "json": { - "upstreams": [ - { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.analytics.analytics.sales_revenue,PROD)", - "type": "TRANSFORMED" - } + "typeNames": [ + "Table" ] } }, @@ -819,6 +791,31 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,hr_pbi_test.revenue,DEV)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,reporting-db.analytics.analytics.sales_revenue,PROD)", + "type": "TRANSFORMED" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", @@ -1395,7 +1392,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset" + "Semantic Model" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json new file mode 100644 index 0000000000000..f8c0fdc17c880 --- /dev/null +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_personal_ingest.json @@ -0,0 +1,329 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "powerbi", + "workspace": "Jane Smith Workspace" + }, + "name": "Jane Smith Workspace" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:powerbi" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PersonalGroup" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User1@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/Jane Smith Workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/customProperties/chartCount", + "value": "0" + }, + { + "op": "add", + "path": "/customProperties/workspaceName", + "value": "Jane Smith Workspace" + }, + { + "op": "add", + "path": "/customProperties/workspaceId", + "value": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C" + }, + { + "op": "add", + "path": "/title", + "value": "test_dashboard" + }, + { + "op": "add", + "path": "/description", + "value": "Description of test dashboard" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://localhost/dashboards/web/1" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User2@foo.com", + "changeType": "UPSERT", + "aspectName": "corpUserKey", + "aspect": { + "json": { + "username": "User2@foo.com" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:users.User1@foo.com", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:users.User1@foo.com", + "type": "NONE" + }, + { + "owner": "urn:li:corpuser:users.User2@foo.com", + "type": "NONE" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a", + "urn": "urn:li:container:4719aeafb92339db2b69194bcbe55c9a" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json index ea1ee0df4b105..6da5f5781112e 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_platform_instance_ingest.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -213,8 +211,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -288,8 +285,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -363,8 +359,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -438,8 +433,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -513,8 +507,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -588,8 +581,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -663,8 +655,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json index 580a8d1a1db11..b8963a0d7782d 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_profiling.json @@ -48,8 +48,8 @@ "json": { "timestampMillis": 1645599600000, "partitionSpec": { - "type": "FULL_TABLE", - "partition": "FULL_TABLE_SNAPSHOT" + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" }, "rowCount": 542300, "columnCount": 4, @@ -115,8 +115,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json index 094869bfd24f1..f6248db9008af 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_report.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_report.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -213,8 +211,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -288,8 +285,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -363,8 +359,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -438,8 +433,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -513,8 +507,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -588,8 +581,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -663,8 +655,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -910,11 +901,12 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "chartKey", + "aspectName": "subTypes", "aspect": { "json": { - "dashboardTool": "powerbi", - "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" + "typeNames": [ + "PowerBI Tile" + ] } }, "systemMetadata": { @@ -927,12 +919,11 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartKey", "aspect": { "json": { - "typeNames": [ - "PowerBI Tile" - ] + "dashboardTool": "powerbi", + "chartId": "charts.23212598-23b5-4980-87cc-5fc0ecd84385" } }, "systemMetadata": { @@ -1213,8 +1204,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1288,8 +1278,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1363,8 +1352,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1438,8 +1426,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1513,8 +1500,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1588,8 +1574,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1663,8 +1648,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -1904,11 +1888,11 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", "changeType": "UPSERT", - "aspectName": "browsePaths", + "aspectName": "subTypes", "aspect": { "json": { - "paths": [ - "/powerbi/demo-workspace" + "typeNames": [ + "PowerBI Page" ] } }, @@ -1922,11 +1906,11 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(powerbi,pages.5b218778-e7a5-4d73-8187-f10824047715.ReportSection1)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "browsePaths", "aspect": { "json": { - "typeNames": [ - "PowerBI Page" + "paths": [ + "/powerbi/demo-workspace" ] } }, @@ -2129,6 +2113,657 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "dummy", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "public issue_history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Snowflake.Databases(\"hp123rt5.ap-southeast-2.fakecomputing.com\",\"PBI_TEST_WAREHOUSE_PROD\",[Role=\"PBI_TEST_MEMBER\"]),\n PBI_TEST_Database = Source{[Name=\"PBI_TEST\",Kind=\"Database\"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name=\"TEST\",Kind=\"Schema\"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name=\"TESTTABLE\",Kind=\"Table\"]}[Data]\nin\n TESTTABLE_Table", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "SNOWFLAKE_TESTTABLE", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.SNOWFLAKE_TESTTABLE,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"bu20658.ap-southeast-2.snowflakecomputing.com\",\"operations_analytics_warehouse_prod\",[Role=\"OPERATIONS_ANALYTICS_MEMBER\"]){[Name=\"OPERATIONS_ANALYTICS\"]}[Data], \"SELECT#(lf)concat((UPPER(REPLACE(SELLER,'-',''))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,'-',''))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4\", null, [EnableFolding=true]),\n #\"Added Conditional Column\" = Table.AddColumn(Source, \"SME Units ENT\", each if [DEAL_TYPE] = \"SME Unit\" then [UNIT] else 0),\n #\"Added Conditional Column1\" = Table.AddColumn(#\"Added Conditional Column\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" then [UNIT] else 0),\n #\"Removed Columns\" = Table.RemoveColumns(#\"Added Conditional Column1\",{\"Banklink Units\"}),\n #\"Added Custom\" = Table.AddColumn(#\"Removed Columns\", \"Banklink Units\", each if [DEAL_TYPE] = \"Banklink\" and [SALES_TYPE] = \"3 - Upsell\"\nthen [UNIT]\n\nelse if [SALES_TYPE] = \"Adjusted BL Migration\"\nthen [UNIT]\n\nelse 0),\n #\"Added Custom1\" = Table.AddColumn(#\"Added Custom\", \"SME Units in $ (*$361)\", each if [DEAL_TYPE] = \"SME Unit\" \nand [SALES_TYPE] <> \"4 - Renewal\"\n then [UNIT] * 361\nelse 0),\n #\"Added Custom2\" = Table.AddColumn(#\"Added Custom1\", \"Banklink in $ (*$148)\", each [Banklink Units] * 148)\nin\n #\"Added Custom2\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = GoogleBigQuery.Database([BillingProject = #\"Parameter - Source\"]),\n#\"gcp-project\" = Source{[Name=#\"Parameter - Source\"]}[Data],\nuniversal_Schema = #\"gcp-project\"{[Name=\"universal\",Kind=\"Schema\"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name=\"D_WH_DATE\",Kind=\"Table\"]}[Data],\n#\"Filtered Rows\" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#\"Filtered Rows1\" = Table.SelectRows(#\"Filtered Rows\", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#\"Filtered Rows1\"", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "big-query-with-parameter", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.big-query-with-parameter,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Value.NativeQuery(Snowflake.Databases(\"xaa48144.snowflakecomputing.com\",\"GSL_TEST_WH\",[Role=\"ACCOUNTADMIN\"]){[Name=\"GSL_TEST_DB\"]}[Data], \"select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, 'mo')\", null, [EnableFolding=true])\nin\n Source", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "snowflake native-query-with-join", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.snowflake_native-query-with-join,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = Oracle.Database(\"localhost:1521/salesdb.domain.com\", [HierarchicalNavigation=true]), HR = Source{[Schema=\"HR\"]}[Data], EMPLOYEES1 = HR{[Name=\"EMPLOYEES\"]}[Data] \n in EMPLOYEES1", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "job-history", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "let\n Source = PostgreSQL.Database(\"localhost\" , \"mics\" ),\n public_order_date = Source{[Schema=\"public\",Item=\"order_date\"]}[Data] \n in \n public_order_date", + "viewLanguage": "m_query" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445" + }, + "externalUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/details", + "name": "postgres_test_table", + "description": "Library dataset description", + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.postgres_test_table,DEV)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "browsePaths", + "aspect": { + "json": { + "paths": [ + "/powerbi/demo-workspace" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "PATCH", + "aspectName": "dashboardInfo", + "aspect": { + "json": [ + { + "op": "add", + "path": "/title", + "value": "Printable SalesMarketing" + }, + { + "op": "add", + "path": "/description", + "value": "Acryl sales marketing report" + }, + { + "op": "add", + "path": "/dashboardUrl", + "value": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163" + }, + { + "op": "add", + "path": "/lastModified", + "value": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "dashboardKey", + "aspect": { + "json": { + "dashboardTool": "powerbi", + "dashboardId": "powerbi.linkedin.com/dashboards/584cf13a-1485-41c2-a514-b1bb66fff163" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "PaginatedReport" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(powerbi,reports.584cf13a-1485-41c2-a514-b1bb66fff163)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "demo-workspace" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "powerbi-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "corpuser", "entityUrn": "urn:li:corpuser:users.User1@foo.com", diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json index dcaa518a3c323..e327ca695beb7 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -213,8 +211,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -288,8 +285,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -363,8 +359,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -438,8 +433,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -513,8 +507,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -588,8 +581,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -663,8 +655,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json index bc5e844f679c7..90c8ee5d0379e 100644 --- a/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json +++ b/metadata-ingestion/tests/integration/powerbi/golden_test_server_to_platform_instance.json @@ -63,8 +63,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -138,8 +137,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -238,8 +236,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -338,8 +335,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -438,8 +434,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -546,8 +541,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -646,8 +640,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -746,8 +739,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, @@ -846,8 +838,7 @@ "aspect": { "json": { "typeNames": [ - "PowerBI Dataset Table", - "View" + "Table" ] } }, diff --git a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py index 91c6082524389..b636c12cfda06 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py +++ b/metadata-ingestion/tests/integration/powerbi/test_admin_only_api.py @@ -58,21 +58,28 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None "status_code": 200, "json": admin_datasets_response, }, - "https://api.powerbi.com/v1.0/myorg/admin/groups": { + "https://api.powerbi.com/v1.0/myorg/admin/groups?%24skip=0&%24top=1000": { "method": "GET", "status_code": 200, "json": { - "@odata.count": 3, "value": [ { "id": "64ED5CAD-7C10-4684-8180-826122881108", "isReadOnly": True, "name": "demo-workspace", "type": "Workspace", + "state": "Active", } ], }, }, + "https://api.powerbi.com/v1.0/myorg/admin/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [], + }, + }, "https://api.powerbi.com/v1.0/myorg/admin/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { "method": "GET", "status_code": 200, @@ -220,6 +227,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None { "id": "64ED5CAD-7C10-4684-8180-826122881108", "name": "demo-workspace", + "type": "Workspace", "state": "Active", "datasets": [ { @@ -391,6 +399,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PowerBIReport", "name": "SalesMarketing", "description": "Acryl sales marketing report", } @@ -422,6 +431,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", "name": "SalesMarketing", + "reportType": "PowerBIReport", "description": "Acryl sales marketing report", "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", @@ -436,6 +446,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", "name": "SalesMarketing", + "reportType": "PowerBIReport", "description": "Acryl sales marketing report", "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", diff --git a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py index 23b23ecada0d4..43f77b059e41f 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_powerbi.py +++ b/metadata-ingestion/tests/integration/powerbi/test_powerbi.py @@ -19,6 +19,7 @@ from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import ( Page, Report, + ReportType, Workspace, ) from tests.test_helpers import mce_helpers, test_connection_helpers @@ -70,6 +71,9 @@ def scan_init_response(request, context): "64ED5CAD-7C10-4684-8180-826122881108||64ED5CAD-7C22-4684-8180-826122881108": { "id": "a674efd1-603c-4129-8d82-03cf2be05aff" }, + "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C": { + "id": "4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3" + }, } return w_id_vs_response[workspace_id] @@ -78,11 +82,10 @@ def scan_init_response(request, context): def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -> None: override_data = override_data or {} api_vs_response = { - "https://api.powerbi.com/v1.0/myorg/groups": { + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { "method": "GET", "status_code": 200, "json": { - "@odata.count": 3, "value": [ { "id": "64ED5CAD-7C10-4684-8180-826122881108", @@ -105,6 +108,13 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - ], }, }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [], + }, + }, "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { "method": "GET", "status_code": 200, @@ -228,6 +238,11 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - ] }, }, + "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": { + "method": "GET", + "status_code": 200, + "json": {"value": []}, + }, "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/tiles": { "method": "GET", "status_code": 200, @@ -318,6 +333,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "id": "64ED5CAD-7C10-4684-8180-826122881108", "name": "demo-workspace", "state": "Active", + "type": "Workspace", "datasets": [ { "id": "05169CD2-E713-41E6-9600-1D8066D95445", @@ -473,6 +489,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PaginatedReport", "name": "SalesMarketing", "description": "Acryl sales marketing report", } @@ -489,6 +506,7 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "id": "64ED5CAD-7C22-4684-8180-826122881108", "name": "second-demo-workspace", + "type": "Workspace", "state": "Active", "datasets": [ { @@ -515,9 +533,17 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PowerBIReport", "name": "SalesMarketing", "description": "Acryl sales marketing report", - } + }, + { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "584cf13a-1485-41c2-a514-b1bb66fff163", + "reportType": "PaginatedReport", + "name": "SalesMarketing", + "description": "Acryl sales marketing report", + }, ], }, ] @@ -536,11 +562,21 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PowerBIReport", "name": "SalesMarketing", "description": "Acryl sales marketing report", "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", - } + }, + { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "584cf13a-1485-41c2-a514-b1bb66fff163", + "reportType": "PaginatedReport", + "name": "Printable SalesMarketing", + "description": "Acryl sales marketing report", + "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163", + "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=584cf13a-1485-41c2-a514-b1bb66fff163&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", + }, ] }, }, @@ -550,12 +586,26 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - "json": { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PowerBIReport", "name": "SalesMarketing", "description": "Acryl sales marketing report", "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715", "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", }, }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/584cf13a-1485-41c2-a514-b1bb66fff163": { + "method": "GET", + "status_code": 200, + "json": { + "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", + "id": "584cf13a-1485-41c2-a514-b1bb66fff163", + "reportType": "PaginatedReport", + "name": "Printable SalesMarketing", + "description": "Acryl sales marketing report", + "webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/584cf13a-1485-41c2-a514-b1bb66fff163", + "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=584cf13a-1485-41c2-a514-b1bb66fff163&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48", + }, + }, "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": { "method": "GET", "status_code": 200, @@ -574,6 +624,11 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - ] }, }, + "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/584cf13a-1485-41c2-a514-b1bb66fff163/pages": { + "method": "GET", + "status_code": 400, # Pages API is not supported for PaginatedReport + "text": '{"error":{"code":"InvalidRequest","message":"Request is currently not supported for RDL reports"}}', + }, "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/parameters": { "method": "GET", "status_code": 200, @@ -612,7 +667,8 @@ def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) - request_mock.register_uri( api_vs_response[url]["method"], url, - json=api_vs_response[url]["json"], + json=api_vs_response[url].get("json"), + text=api_vs_response[url].get("text"), status_code=api_vs_response[url]["status_code"], ) @@ -683,6 +739,131 @@ def test_powerbi_ingest( ) +@freeze_time(FROZEN_TIME) +@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) +@pytest.mark.integration +def test_powerbi_workspace_type_filter( + mock_msal: MagicMock, + pytestconfig: pytest.Config, + tmp_path: str, + mock_time: datetime.datetime, + requests_mock: Any, +) -> None: + enable_logging() + + test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi" + + register_mock_api( + request_mock=requests_mock, + override_data={ + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C", + "isReadOnly": True, + "name": "Jane Smith Workspace", + "type": "PersonalGroup", + "state": "Active", + }, + { + "id": "C6B5DBBC-7580-406C-A6BE-72628C28801C", + "isReadOnly": True, + "name": "Sales", + "type": "Workspace", + "state": "Active", + }, + ], + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [], + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": { + "method": "GET", + "status_code": 200, + "json": { + "workspaces": [ + { + "id": "90E9E256-3D6D-4D38-86C8-6CCCBD8C170C", + "name": "Jane Smith Workspace", + "type": "PersonalGroup", + "state": "Active", + "datasets": [], + }, + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/groups/90E9E256-3D6D-4D38-86C8-6CCCBD8C170C/dashboards": { + "method": "GET", + "status_code": 200, + "json": { + "value": [ + { + "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", + "isReadOnly": True, + "displayName": "test_dashboard", + "description": "Description of test dashboard", + "embedUrl": "https://localhost/dashboards/embed/1", + "webUrl": "https://localhost/dashboards/web/1", + } + ] + }, + }, + "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4278EDC0-85AA-4BF2-B96A-2BC6C82B73C3": { + "method": "GET", + "status_code": 200, + "json": { + "status": "SUCCEEDED", + }, + }, + }, + ) + + default_config: dict = default_source_config() + + del default_config["workspace_id"] + del default_config["workspace_id_pattern"] + + pipeline = Pipeline.create( + { + "run_id": "powerbi-test", + "source": { + "type": "powerbi", + "config": { + **default_config, + "extract_workspaces_to_containers": True, + "workspace_type_filter": [ + "PersonalGroup", + ], + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/powerbi_mces.json", + }, + }, + } + ) + + pipeline.run() + pipeline.raise_from_status() + golden_file = "golden_test_personal_ingest.json" + + mce_helpers.check_golden_file( + pytestconfig, + output_path=f"{tmp_path}/powerbi_mces.json", + golden_path=f"{test_resources_dir}/{golden_file}", + ) + + @freeze_time(FROZEN_TIME) @mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca) @pytest.mark.integration @@ -1439,6 +1620,7 @@ def validate_pipeline(pipeline: Pipeline) -> None: mock_workspace: Workspace = Workspace( id="64ED5CAD-7C10-4684-8180-826122881108", name="demo-workspace", + type="Workspace", datasets={}, dashboards=[], reports=[], @@ -1485,6 +1667,7 @@ def validate_pipeline(pipeline: Pipeline) -> None: Report( id=report[Constant.ID], name=report[Constant.NAME], + type=ReportType.PowerBIReport, webUrl="", embedUrl="", description=report[Constant.DESCRIPTION], @@ -1538,6 +1721,7 @@ def test_reports_with_failed_page_request( { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", + "reportType": "PowerBIReport", "name": "SalesMarketing", "description": "Acryl sales marketing report", "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715", @@ -1546,6 +1730,7 @@ def test_reports_with_failed_page_request( { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "e9fd6b0b-d8c8-4265-8c44-67e183aebf97", + "reportType": "PaginatedReport", "name": "Product", "description": "Acryl product report", "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97", @@ -1561,6 +1746,7 @@ def test_reports_with_failed_page_request( "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "5b218778-e7a5-4d73-8187-f10824047715", "name": "SalesMarketing", + "reportType": "PowerBIReport", "description": "Acryl sales marketing report", "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715", "embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=64ED5CAD-7C10-4684-8180-826122881108", @@ -1572,6 +1758,7 @@ def test_reports_with_failed_page_request( "json": { "datasetId": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "e9fd6b0b-d8c8-4265-8c44-67e183aebf97", + "reportType": "PowerBIReport", "name": "Product", "description": "Acryl product report", "webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97", @@ -1647,11 +1834,10 @@ def test_independent_datasets_extraction( register_mock_api( request_mock=requests_mock, override_data={ - "https://api.powerbi.com/v1.0/myorg/groups": { + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { "method": "GET", "status_code": 200, "json": { - "@odata.count": 3, "value": [ { "id": "64ED5CAD-7C10-4684-8180-826122881108", @@ -1662,6 +1848,13 @@ def test_independent_datasets_extraction( ], }, }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [], + }, + }, "https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": { "method": "GET", "status_code": 200, @@ -1670,6 +1863,7 @@ def test_independent_datasets_extraction( { "id": "64ED5CAD-7C10-4684-8180-826122881108", "name": "demo-workspace", + "type": "Workspace", "state": "Active", "datasets": [ { diff --git a/metadata-ingestion/tests/integration/powerbi/test_profiling.py b/metadata-ingestion/tests/integration/powerbi/test_profiling.py index 7955386de8940..4b48bed003b1e 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_profiling.py +++ b/metadata-ingestion/tests/integration/powerbi/test_profiling.py @@ -112,21 +112,28 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None "status_code": 200, "json": admin_datasets_response, }, - "https://api.powerbi.com/v1.0/myorg/groups?%24top=1000&%24skip=0&%24filter=type+eq+%27Workspace%27": { + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { "method": "GET", "status_code": 200, "json": { - "@odata.count": 3, "value": [ { "id": "64ED5CAD-7C10-4684-8180-826122881108", "isReadOnly": True, "name": "demo-workspace", "type": "Workspace", + "state": "Active", } ], }, }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [], + }, + }, "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { "method": "GET", "status_code": 200, @@ -176,6 +183,7 @@ def register_mock_admin_api(request_mock: Any, override_data: dict = {}) -> None "id": "64ED5CAD-7C10-4684-8180-826122881108", "name": "demo-workspace", "state": "Active", + "type": "Workspace", "datasets": [ { "id": "05169CD2-E713-41E6-9600-1D8066D95445", diff --git a/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py b/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py index 077b48ca177b5..84f7a87ce5d2d 100644 --- a/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py +++ b/metadata-ingestion/tests/integration/powerbi/test_stateful_ingestion.py @@ -23,27 +23,35 @@ def register_mock_api_state1(request_mock): "status_code": 403, "json": {}, }, - "https://api.powerbi.com/v1.0/myorg/groups": { + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { "method": "GET", "status_code": 200, "json": { - "@odata.count": 1, "value": [ { "id": "64ED5CAD-7C10-4684-8180-826122881108", "isReadOnly": True, "name": "Workspace 1", "type": "Workspace", + "state": "Active", }, { "id": "44444444-7C10-4684-8180-826122881108", "isReadOnly": True, "name": "Multi Workspace", "type": "Workspace", + "state": "Active", }, ], }, }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [], + }, + }, "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { "method": "GET", "status_code": 200, @@ -114,7 +122,7 @@ def register_mock_api_state2(request_mock): "status_code": 403, "json": {}, }, - "https://api.powerbi.com/v1.0/myorg/groups": { + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=0&%24top=1000": { "method": "GET", "status_code": 200, "json": { @@ -135,6 +143,13 @@ def register_mock_api_state2(request_mock): ], }, }, + "https://api.powerbi.com/v1.0/myorg/groups?%24skip=1000&%24top=1000": { + "method": "GET", + "status_code": 200, + "json": { + "value": [], + }, + }, "https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": { "method": "GET", "status_code": 200, From 45b0693b8ebe3cd3f9629a7b7f9f359b5566d213 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Fri, 11 Oct 2024 08:55:40 -0700 Subject: [PATCH 09/17] fix(ingest): gracefully handle missing system metadata in client (#11592) --- metadata-ingestion/src/datahub/ingestion/graph/client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index b9b0ed556e66c..e8fae6254ae88 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -559,8 +559,10 @@ def get_entity_as_mcps( post_json_obj = post_json_transform(aspect_json) aspect_value = aspect_type.from_obj(post_json_obj["value"]) - system_metadata_raw = post_json_obj["systemMetadata"] - system_metadata = SystemMetadataClass.from_obj(system_metadata_raw) + system_metadata_raw = post_json_obj.get("systemMetadata") + system_metadata = None + if system_metadata_raw: + system_metadata = SystemMetadataClass.from_obj(system_metadata_raw) mcpw = MetadataChangeProposalWrapper( entityUrn=entity_urn, @@ -590,7 +592,7 @@ def get_entity_semityped( not be present in the dictionary. The entity's key aspect will always be present. """ - mcps = self.get_entity_as_mcps(entity_urn, aspects) + mcps = self.get_entity_as_mcps(entity_urn, aspects=aspects) result: AspectBag = {} for mcp in mcps: From 089f447d9587e40121587b89edd33c585fd408ed Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Fri, 11 Oct 2024 09:20:14 -0700 Subject: [PATCH 10/17] docs(assertions): add example of fetching associated dataset to assertion docs (#11566) --- docs/api/tutorials/custom-assertions.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/api/tutorials/custom-assertions.md b/docs/api/tutorials/custom-assertions.md index 6544efb8809c2..47975c5739464 100644 --- a/docs/api/tutorials/custom-assertions.md +++ b/docs/api/tutorials/custom-assertions.md @@ -265,7 +265,7 @@ query getAssertion { customType # Will be your custom type. description lastUpdated { - time + time actor } customAssertion { @@ -282,6 +282,18 @@ query getAssertion { } } } + # Fetch what entities have the assertion attached to it + relationships(input: { + types: ["Asserts"] + direction: OUTGOING + }) { + total + relationships { + entity { + urn + } + } + } } } ``` From 14c79389f580e8e4641c8ca60f4ee0e68f2897e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20L=C3=BCdin?= <13187726+Masterchen09@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:40:05 +0200 Subject: [PATCH 11/17] fix(ingest/sac): handle descriptions which are None correctly (#11572) --- .../src/datahub/ingestion/source/sac/sac.py | 35 +++++++++++++++---- .../ingestion/source/sac/sac_common.py | 6 ++-- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py index 8309c469f67c5..de0904107b9bb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py @@ -329,7 +329,9 @@ def get_resource_workunits( entityUrn=dashboard_urn, aspect=DashboardInfoClass( title=resource.name, - description=resource.description, + description=resource.description + if resource.description is not None + else "", lastModified=ChangeAuditStampsClass( created=AuditStampClass( time=round(resource.created_time.timestamp() * 1000), @@ -559,7 +561,14 @@ def get_sac_connection( retries = 3 backoff_factor = 10 - status_forcelist = (500,) + + # The Resources and Data Import Service APIs of SAP Analytics Cloud can be somewhat unstable, occasionally + # returning HTTP errors for some requests, even though the APIs are generally operational. Therefore, we must + # retry these requests to increase the likelihood that the ingestion is successful. For the same reason we + # should also retry requests that receive a 401 HTTP status; however, this status also legitimately indicates + # that the provided OAuth credentials are invalid or that the OAuth client does not have the correct + # permissions assigned, therefore requests that receive a 401 HTTP status must not be retried. + status_forcelist = (400, 500, 503) retry = Retry( total=retries, @@ -611,7 +620,9 @@ def get_resources(self) -> Iterable[Resource]: entity: pyodata.v2.service.EntityProxy for entity in entities: resource_id: str = entity.resourceId - name: str = entity.name.strip() + name: str = ( + entity.name.strip() if entity.name is not None else entity.resourceId + ) if not self.config.resource_id_pattern.allowed( resource_id @@ -655,8 +666,12 @@ def get_resources(self) -> Iterable[Resource]: ResourceModel( namespace=namespace, model_id=model_id, - name=nav_entity.name.strip(), - description=nav_entity.description.strip(), + name=nav_entity.name.strip() + if nav_entity.name is not None + else f"{namespace}:{model_id}", + description=nav_entity.description.strip() + if nav_entity.description is not None + else None, system_type=nav_entity.systemType, # BW or HANA connection_id=nav_entity.connectionId, external_id=nav_entity.externalId, # query:[][][query] or view:[schema][schema.namespace][view] @@ -678,7 +693,9 @@ def get_resources(self) -> Iterable[Resource]: resource_subtype=entity.resourceSubtype, story_id=entity.storyId, name=name, - description=entity.description.strip(), + description=entity.description.strip() + if entity.description is not None + else None, created_time=entity.createdTime, created_by=created_by, modified_time=entity.modifiedTime, @@ -715,7 +732,11 @@ def get_import_data_model_columns( columns.append( ImportDataModelColumn( name=column["columnName"].strip(), - description=column["descriptionName"].strip(), + description=( + column["descriptionName"].strip() + if column.get("descriptionName") is not None + else None + ), property_type=column["propertyType"], data_type=column["columnDataType"], max_length=column.get("maxLength"), diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py index 457fda1e06181..2c02b444cea1c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac_common.py @@ -8,7 +8,7 @@ class ResourceModel: namespace: str model_id: str name: str - description: str + description: Optional[str] system_type: Optional[str] connection_id: Optional[str] external_id: Optional[str] @@ -22,7 +22,7 @@ class Resource: resource_subtype: str story_id: str name: str - description: str + description: Optional[str] created_time: datetime created_by: Optional[str] modified_time: datetime @@ -36,7 +36,7 @@ class Resource: @dataclass(frozen=True) class ImportDataModelColumn: name: str - description: str + description: Optional[str] property_type: str data_type: str max_length: Optional[int] From ebcce1f87bddf2bf8739b7060a046ac0c80e1fe2 Mon Sep 17 00:00:00 2001 From: skrydal Date: Fri, 11 Oct 2024 18:48:25 +0200 Subject: [PATCH 12/17] fix(ingest/iceberg): Iceberg table name (#11599) --- .../src/datahub/ingestion/source/iceberg/iceberg.py | 1 + metadata-ingestion/tests/integration/iceberg/docker-compose.yml | 2 -- .../integration/iceberg/iceberg_deleted_table_mces_golden.json | 1 + .../tests/integration/iceberg/iceberg_ingest_mces_golden.json | 1 + .../tests/integration/iceberg/iceberg_profile_mces_golden.json | 1 + 5 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py index b5caa83b2ff37..d8c6c03ce81e6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py @@ -182,6 +182,7 @@ def _create_iceberg_workunit( custom_properties["snapshot-id"] = str(table.current_snapshot().snapshot_id) custom_properties["manifest-list"] = table.current_snapshot().manifest_list dataset_properties = DatasetPropertiesClass( + name=table.name()[-1], tags=[], description=table.metadata.properties.get("comment", None), customProperties=custom_properties, diff --git a/metadata-ingestion/tests/integration/iceberg/docker-compose.yml b/metadata-ingestion/tests/integration/iceberg/docker-compose.yml index 8baae6e8ab636..8a05ac7481fe2 100644 --- a/metadata-ingestion/tests/integration/iceberg/docker-compose.yml +++ b/metadata-ingestion/tests/integration/iceberg/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - services: spark-iceberg: image: tabulario/spark-iceberg:3.3.2_1.3.0 diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json index 3321fcac0d73e..4b2afb29ddda8 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_deleted_table_mces_golden.json @@ -11,6 +11,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "name": "another_taxis", "customProperties": { "owner": "root", "created-at": "2024-06-27T17:29:32.492204247Z", diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json index b017b6cd31520..477f719ef9317 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_ingest_mces_golden.json @@ -11,6 +11,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "name": "taxis", "customProperties": { "owner": "root", "created-at": "2024-05-22T14:08:04.001538500Z", diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json index 453a79494fa25..6d2ca013d81d0 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/iceberg_profile_mces_golden.json @@ -11,6 +11,7 @@ }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "name": "taxis", "customProperties": { "owner": "root", "created-at": "2024-05-22T14:10:22.926080700Z", From 4b66757a62a9f2f6d9a713948bfa079f760ccc5d Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 11 Oct 2024 11:52:52 -0500 Subject: [PATCH 13/17] chore(frontend): force frontend protobuf version (#11601) --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 79a4ca9384d28..67968ce3ee290 100644 --- a/build.gradle +++ b/build.gradle @@ -398,6 +398,7 @@ subprojects { implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") implementation("com.fasterxml.jackson.core:jackson-dataformat-cbor:$jacksonVersion") implementation(externalDependency.commonsIo) + implementation(externalDependency.protobuf) } } From f13dae1cd2f6659564755bf2bb5a4ddb70ab1a87 Mon Sep 17 00:00:00 2001 From: dushayntAW <158567391+dushayntAW@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:46:01 +0200 Subject: [PATCH 14/17] fix(airflow): add dag AllowDenyPattern config (#11472) Co-authored-by: Harshal Sheth --- docs/lineage/airflow.md | 3 +- .../src/datahub_airflow_plugin/_config.py | 12 ++++++- .../datahub_listener.py | 18 ++++++++-- .../tests/integration/dags/dag_to_skip.py | 34 ++++++++++++++++++ .../tests/integration/test_plugin.py | 36 +++++++++++-------- 5 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py diff --git a/docs/lineage/airflow.md b/docs/lineage/airflow.md index aca6d30619ea8..35f2ff862e695 100644 --- a/docs/lineage/airflow.md +++ b/docs/lineage/airflow.md @@ -132,7 +132,7 @@ conn_id = datahub_rest_default # or datahub_kafka_default ``` | Name | Default value | Description | -| -------------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|----------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | enabled | true | If the plugin should be enabled. | | conn_id | datahub_rest_default | The name of the datahub connection you set in step 1. | | cluster | prod | name of the airflow cluster | @@ -145,6 +145,7 @@ conn_id = datahub_rest_default # or datahub_kafka_default | datajob_url_link | taskinstance | If taskinstance, the datajob url will be taskinstance link on airflow. It can also be grid. | | | | graceful_exceptions | true | If set to true, most runtime errors in the lineage backend will be suppressed and will not cause the overall task to fail. Note that configuration issues will still throw exceptions. | +| dag_filter_str | { "allow": [".*"] } | AllowDenyPattern value in form of JSON string to filter the DAGs from running. | #### Validate that the plugin is working diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py index 8deba22a107ce..c4964712cf9f7 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_config.py @@ -3,7 +3,8 @@ import datahub.emitter.mce_builder as builder from airflow.configuration import conf -from datahub.configuration.common import ConfigModel +from datahub.configuration.common import AllowDenyPattern, ConfigModel +from pydantic.fields import Field if TYPE_CHECKING: from datahub_airflow_plugin.hooks.datahub import DatahubGenericHook @@ -56,6 +57,11 @@ class DatahubLineageConfig(ConfigModel): # Makes extraction of jinja-templated fields more accurate. render_templates: bool = True + dag_filter_pattern: AllowDenyPattern = Field( + default=AllowDenyPattern.allow_all(), + description="regex patterns for DAGs to ingest", + ) + log_level: Optional[str] = None debug_emitter: bool = False @@ -93,6 +99,9 @@ def get_lineage_config() -> DatahubLineageConfig: datajob_url_link = conf.get( "datahub", "datajob_url_link", fallback=DatajobUrl.TASKINSTANCE.value ) + dag_filter_pattern = AllowDenyPattern.parse_raw( + conf.get("datahub", "dag_filter_str", fallback='{"allow": [".*"]}') + ) return DatahubLineageConfig( enabled=enabled, @@ -109,4 +118,5 @@ def get_lineage_config() -> DatahubLineageConfig: disable_openlineage_plugin=disable_openlineage_plugin, datajob_url_link=datajob_url_link, render_templates=render_templates, + dag_filter_pattern=dag_filter_pattern, ) diff --git a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py index b818b76de9f7f..d1c7e996dd03d 100644 --- a/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py +++ b/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_listener.py @@ -383,9 +383,15 @@ def on_task_instance_running( return logger.debug( - f"DataHub listener got notification about task instance start for {task_instance.task_id}" + f"DataHub listener got notification about task instance start for {task_instance.task_id} of dag {task_instance.dag_run.dag_id}" ) + if not self.config.dag_filter_pattern.allowed(task_instance.dag_run.dag_id): + logger.debug( + f"DAG {task_instance.dag_run.dag_id} is not allowed by the pattern" + ) + return + if self.config.render_templates: task_instance = _render_templates(task_instance) @@ -492,6 +498,10 @@ def on_task_instance_finish( dag: "DAG" = task.dag # type: ignore[assignment] + if not self.config.dag_filter_pattern.allowed(dag.dag_id): + logger.debug(f"DAG {dag.dag_id} is not allowed by the pattern") + return + datajob = AirflowGenerator.generate_datajob( cluster=self.config.cluster, task=task, @@ -689,8 +699,12 @@ def on_dag_run_running(self, dag_run: "DagRun", msg: str) -> None: f"DataHub listener got notification about dag run start for {dag_run.dag_id}" ) - self.on_dag_start(dag_run) + assert dag_run.dag_id + if not self.config.dag_filter_pattern.allowed(dag_run.dag_id): + logger.debug(f"DAG {dag_run.dag_id} is not allowed by the pattern") + return + self.on_dag_start(dag_run) self.emitter.flush() # TODO: Add hooks for on_dag_run_success, on_dag_run_failed -> call AirflowGenerator.complete_dataflow diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py new file mode 100644 index 0000000000000..a805a2219d142 --- /dev/null +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/dag_to_skip.py @@ -0,0 +1,34 @@ +from datetime import datetime + +from airflow import DAG +from airflow.operators.bash import BashOperator + +from datahub_airflow_plugin.entities import Dataset, Urn + +with DAG( + "dag_to_skip", + start_date=datetime(2023, 1, 1), + schedule_interval=None, + catchup=False, +) as dag: + task1 = BashOperator( + task_id="dag_to_skip_task_1", + dag=dag, + bash_command="echo 'dag_to_skip_task_1'", + inlets=[ + Dataset(platform="snowflake", name="mydb.schema.tableA"), + Urn( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,mydb.schema.tableC,PROD)" + ), + Urn("urn:li:dataJob:(urn:li:dataFlow:(airflow,test_dag,PROD),test_task)"), + ], + outlets=[Dataset("snowflake", "mydb.schema.tableD")], + ) + + task2 = BashOperator( + task_id="dag_to_skip_task_2", + dag=dag, + bash_command="echo 'dag_to_skip_task_2'", + ) + + task1 >> task2 diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py index 37cd3b792d535..44efd94f834b1 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py @@ -33,6 +33,8 @@ DAGS_FOLDER = pathlib.Path(__file__).parent / "dags" GOLDENS_FOLDER = pathlib.Path(__file__).parent / "goldens" +DAG_TO_SKIP_INGESTION = "dag_to_skip" + @dataclasses.dataclass class AirflowInstance: @@ -140,6 +142,7 @@ def _run_airflow( # Configure the datahub plugin and have it write the MCPs to a file. "AIRFLOW__CORE__LAZY_LOAD_PLUGINS": "False" if is_v1 else "True", "AIRFLOW__DATAHUB__CONN_ID": datahub_connection_name, + "AIRFLOW__DATAHUB__DAG_FILTER_STR": f'{{ "deny": ["{DAG_TO_SKIP_INGESTION}"] }}', f"AIRFLOW_CONN_{datahub_connection_name.upper()}": Connection( conn_id="datahub_file_default", conn_type="datahub-file", @@ -276,6 +279,7 @@ class DagTestCase: test_cases = [ DagTestCase("simple_dag"), DagTestCase("basic_iolets"), + DagTestCase("dag_to_skip", v2_only=True), DagTestCase("snowflake_operator", success=False, v2_only=True), DagTestCase("sqlite_operator", v2_only=True), DagTestCase("custom_operator_dag", v2_only=True), @@ -373,20 +377,24 @@ def test_airflow_plugin( print("Sleeping for a few seconds to let the plugin finish...") time.sleep(10) - _sanitize_output_file(airflow_instance.metadata_file) - - check_golden_file( - pytestconfig=pytestconfig, - output_path=airflow_instance.metadata_file, - golden_path=golden_path, - ignore_paths=[ - # TODO: If we switched to Git urls, maybe we could get this to work consistently. - r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['datahub_sql_parser_error'\]", - r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['openlineage_.*'\]", - r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['log_url'\]", - r"root\[\d+\]\['aspect'\]\['json'\]\['externalUrl'\]", - ], - ) + if dag_id == DAG_TO_SKIP_INGESTION: + # Verify that no MCPs were generated. + assert not os.path.exists(airflow_instance.metadata_file) + else: + _sanitize_output_file(airflow_instance.metadata_file) + + check_golden_file( + pytestconfig=pytestconfig, + output_path=airflow_instance.metadata_file, + golden_path=golden_path, + ignore_paths=[ + # TODO: If we switched to Git urls, maybe we could get this to work consistently. + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['datahub_sql_parser_error'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['openlineage_.*'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['customProperties'\]\['log_url'\]", + r"root\[\d+\]\['aspect'\]\['json'\]\['externalUrl'\]", + ], + ) def _sanitize_output_file(output_path: pathlib.Path) -> None: From 36fc0c4e23ec036af01f07017e0380677c4bef9b Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Fri, 11 Oct 2024 13:23:54 -0700 Subject: [PATCH 15/17] =?UTF-8?q?docs(structured-properties):=20example=20?= =?UTF-8?q?to=20read=20structured=20properties=20fr=E2=80=A6=20(#11603)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api/tutorials/structured-properties.md | 46 ++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/api/tutorials/structured-properties.md b/docs/api/tutorials/structured-properties.md index 00e992f2bd0bb..9b18aa922290b 100644 --- a/docs/api/tutorials/structured-properties.md +++ b/docs/api/tutorials/structured-properties.md @@ -532,6 +532,50 @@ Or you can run the following command to view the properties associated with the datahub dataset get --urn {urn} ``` +## Read Structured Properties From a Dataset + +For reading all structured properties from a dataset: + + + + +```graphql +query getDataset { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.customer,PROD)") { + structuredProperties { + properties { + structuredProperty { + urn + type + definition { + displayName + description + allowedValues { + description + } + } + } + values { + ... on StringValue { + stringValue + } + ... on NumberValue { + numberValue + } + } + valueEntities { + urn + type + } + } + } + } +} +``` + + + + ## Remove Structured Properties From a Dataset For removing a structured property or list of structured properties from a dataset: @@ -1733,4 +1777,4 @@ Example Response: ``` - \ No newline at end of file + From b5cf729c9be29d44e6e499ba3a6971ed23efc0f6 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:29:58 -0500 Subject: [PATCH 16/17] fix(bootstrap): fix early bootstrap mcps (#11605) --- .../configuration/src/main/resources/bootstrap_mcps.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml index b1612f95f9219..10ae176b2c31e 100644 --- a/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml +++ b/metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml @@ -14,18 +14,26 @@ bootstrap: - name: data-platforms version: v1 + blocking: true + async: false mcps_location: "bootstrap_mcps/data-platforms.yaml" - name: data-types version: v1 + blocking: true + async: false mcps_location: "bootstrap_mcps/data-types.yaml" - name: ownership-types version: v1 + blocking: true + async: false mcps_location: "bootstrap_mcps/ownership-types.yaml" - name: roles version: v1 + blocking: true + async: false mcps_location: "bootstrap_mcps/roles.yaml" # Ingestion Recipes From 413331933a9fd7f56931faad50587ebc2b0de4cb Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:08:25 -0500 Subject: [PATCH 17/17] fix(spark-lineage-legacy): fix check jar script (#11608) --- .../java/spark-lineage-legacy/scripts/check_jar.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh index 81d6a541d1c2a..854c4227d08d9 100755 --- a/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh +++ b/metadata-integration/java/spark-lineage-legacy/scripts/check_jar.sh @@ -40,7 +40,8 @@ jar -tvf $jarFile |\ grep -v "rootdoc.txt" |\ grep -v "VersionInfo.java" |\ grep -v "mime.types" |\ - grep -v "com/ibm/.*" + grep -v "com/ibm/.*" |\ + grep -v "google/" if [ $? -ne 0 ]; then