Merge branch 'datahub-project:master' into master

acryldata · Jan 12, 2025 · c6be903 · c6be903
2 parents f9b3282 + 9897804
commit c6be903
Show file tree

Hide file tree

Showing 49 changed files with 1,239 additions and 725 deletions.
diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml
@@ -13,16 +13,16 @@ outputs:
     value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
   backend-change:
     description: "Backend code has changed"
-    value: ${{ steps.filter.outputs.backend == 'true' }}
+    value: ${{ steps.filter.outputs.backend == 'true' ||  steps.trigger.outputs.trigger == 'manual' }}
   ingestion-change:
     description: "Ingestion code has changed"
-    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+    value: ${{ steps.filter.outputs.ingestion == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   ingestion-base-change:
     description: "Ingestion base image docker image has changed"
     value: ${{ steps.filter.outputs.ingestion-base == 'true' }}
   frontend-change:
     description: "Frontend code has changed"
-    value: ${{ steps.filter.outputs.frontend == 'true' }}
+    value: ${{ steps.filter.outputs.frontend == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   docker-change:
     description: "Docker code has changed"
     value: ${{ steps.filter.outputs.docker == 'true' }}
@@ -44,6 +44,15 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: Check trigger type
+      id: trigger  # Add an ID to reference this step
+      shell: bash
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          echo "trigger=manual" >> $GITHUB_OUTPUT
+        else
+          echo "trigger=pr" >> $GITHUB_OUTPUT
+        fi
     - uses: dorny/paths-filter@v3
       id: filter
       with:

diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -12,6 +12,7 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
+  workflow_dispatch:
   release:
     types: [published]
 
@@ -117,10 +118,12 @@ jobs:
         run: |
           echo "BACKEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(metadata-models|entity-registry|datahuyb-graphql-core|metadata-io|metadata-jobs|metadata-utils|metadata-service|medata-dao-impl|metadata-operation|li-utils|metadata-integration|metadata-events|metadata-auth|ingestion-scheduler|notifications|datahub-upgrade)' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
           echo "FRONTEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(datahub-frontend|datahub-web-react).*\.(xml|json)$' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
+      - name: Generate tz artifact name
+        run: echo "NAME_TZ=$(echo ${{ matrix.timezone }} | tr '/' '-')" >> $GITHUB_ENV
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: Test Results (build)
+          name: Test Results (build) - ${{ matrix.command}}-${{ env.NAME_TZ }}
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**

diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
@@ -20,6 +20,7 @@ on:
       - ".github/workflows/metadata-io.yml"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

diff --git a/build.gradle b/build.gradle
@@ -379,6 +379,7 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
 
     resolutionStrategy.force externalDependency.antlr4Runtime
     resolutionStrategy.force externalDependency.antlr4
+    resolutionStrategy.force 'org.apache.mina:mina-core:2.2.4'
   }
 }
 

diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
@@ -2377,6 +2377,17 @@ private void configureDataJobResolvers(final RuntimeWiring.Builder builder) {
                                   ? dataJob.getDataPlatformInstance().getUrn()
                                   : null;
                             }))
+                    .dataFetcher(
+                        "container",
+                        new LoadableTypeResolver<>(
+                            containerType,
+                            (env) -> {
+                              final DataJob dataJob = env.getSource();
+                              return dataJob.getContainer() != null
+                                  ? dataJob.getContainer().getUrn()
+                                  : null;
+                            }))
+                    .dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
                     .dataFetcher("runs", new DataJobRunsResolver(entityClient))
                     .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient))
                     .dataFetcher("exists", new EntityExistsResolver(entityService))
@@ -2454,6 +2465,17 @@ private void configureDataFlowResolvers(final RuntimeWiring.Builder builder) {
                               ? dataFlow.getDataPlatformInstance().getUrn()
                               : null;
                         }))
+                .dataFetcher(
+                    "container",
+                    new LoadableTypeResolver<>(
+                        containerType,
+                        (env) -> {
+                          final DataFlow dataFlow = env.getSource();
+                          return dataFlow.getContainer() != null
+                              ? dataFlow.getContainer().getUrn()
+                              : null;
+                        }))
+                .dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
                 .dataFetcher(
                     "health",
                     new EntityHealthResolver(

diff --git a/...-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/...-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java
@@ -74,6 +74,7 @@ public class DataFlowType
           DOMAINS_ASPECT_NAME,
           DEPRECATION_ASPECT_NAME,
           DATA_PLATFORM_INSTANCE_ASPECT_NAME,
+          CONTAINER_ASPECT_NAME,
           DATA_PRODUCTS_ASPECT_NAME,
           BROWSE_PATHS_V2_ASPECT_NAME,
           STRUCTURED_PROPERTIES_ASPECT_NAME,

diff --git a/...ore/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java b/...ore/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java
@@ -16,6 +16,7 @@
 import com.linkedin.data.DataMap;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
+import com.linkedin.datahub.graphql.generated.Container;
 import com.linkedin.datahub.graphql.generated.DataFlow;
 import com.linkedin.datahub.graphql.generated.DataFlowEditableProperties;
 import com.linkedin.datahub.graphql.generated.DataFlowInfo;
@@ -106,6 +107,7 @@ public DataFlow apply(
         (dataset, dataMap) ->
             dataset.setDataPlatformInstance(
                 DataPlatformInstanceAspectMapper.map(context, new DataPlatformInstance(dataMap))));
+    mappingHelper.mapToResult(context, CONTAINER_ASPECT_NAME, DataFlowMapper::mapContainers);
     mappingHelper.mapToResult(
         BROWSE_PATHS_V2_ASPECT_NAME,
         (dataFlow, dataMap) ->
@@ -206,6 +208,17 @@ private static void mapGlobalTags(
     dataFlow.setTags(globalTags);
   }
 
+  private static void mapContainers(
+      @Nullable final QueryContext context, @Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap) {
+    final com.linkedin.container.Container gmsContainer =
+        new com.linkedin.container.Container(dataMap);
+    dataFlow.setContainer(
+        Container.builder()
+            .setType(EntityType.CONTAINER)
+            .setUrn(gmsContainer.getContainer().toString())
+            .build());
+  }
+
   private static void mapDomains(
       @Nullable final QueryContext context, @Nonnull DataFlow dataFlow, @Nonnull DataMap dataMap) {
     final Domains domains = new Domains(dataMap);

diff --git a/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java
@@ -75,6 +75,7 @@ public class DataJobType
           DOMAINS_ASPECT_NAME,
           DEPRECATION_ASPECT_NAME,
           DATA_PLATFORM_INSTANCE_ASPECT_NAME,
+          CONTAINER_ASPECT_NAME,
           DATA_PRODUCTS_ASPECT_NAME,
           BROWSE_PATHS_V2_ASPECT_NAME,
           SUB_TYPES_ASPECT_NAME,

diff --git a/...-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/...-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java
@@ -9,6 +9,7 @@
 import com.linkedin.data.DataMap;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
+import com.linkedin.datahub.graphql.generated.Container;
 import com.linkedin.datahub.graphql.generated.DataFlow;
 import com.linkedin.datahub.graphql.generated.DataJob;
 import com.linkedin.datahub.graphql.generated.DataJobEditableProperties;
@@ -112,6 +113,14 @@ public DataJob apply(
               } else if (DATA_PLATFORM_INSTANCE_ASPECT_NAME.equals(name)) {
                 result.setDataPlatformInstance(
                     DataPlatformInstanceAspectMapper.map(context, new DataPlatformInstance(data)));
+              } else if (CONTAINER_ASPECT_NAME.equals(name)) {
+                final com.linkedin.container.Container gmsContainer =
+                    new com.linkedin.container.Container(data);
+                result.setContainer(
+                    Container.builder()
+                        .setType(EntityType.CONTAINER)
+                        .setUrn(gmsContainer.getContainer().toString())
+                        .build());
               } else if (BROWSE_PATHS_V2_ASPECT_NAME.equals(name)) {
                 result.setBrowsePathV2(BrowsePathsV2Mapper.map(context, new BrowsePathsV2(data)));
               } else if (SUB_TYPES_ASPECT_NAME.equals(name)) {

diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -6275,6 +6275,16 @@ type DataFlow implements EntityWithRelationships & Entity & BrowsableEntity {
     """
     dataPlatformInstance: DataPlatformInstance
 
+    """
+    The parent container in which the entity resides
+    """
+    container: Container
+
+    """
+    Recursively get the lineage of containers for this entity
+    """
+    parentContainers: ParentContainersResult
+
     """
     Granular API for querying edges extending from this entity
     """
@@ -6457,6 +6467,16 @@ type DataJob implements EntityWithRelationships & Entity & BrowsableEntity {
     """
     dataPlatformInstance: DataPlatformInstance
 
+    """
+    The parent container in which the entity resides
+    """
+    container: Container
+
+    """
+    Recursively get the lineage of containers for this entity
+    """
+    parentContainers: ParentContainersResult
+
     """
     Additional read write properties associated with the Data Job
     """

diff --git a/...src/test/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapperTest.java b/...src/test/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapperTest.java
@@ -0,0 +1,42 @@
+package com.linkedin.datahub.graphql.types.dataflow.mappers;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.generated.DataFlow;
+import com.linkedin.entity.Aspect;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.EnvelopedAspectMap;
+import com.linkedin.metadata.Constants;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class DataFlowMapperTest {
+  private static final Urn TEST_DATA_FLOW_URN =
+      Urn.createFromTuple(Constants.DATA_FLOW_ENTITY_NAME, "dataflow1");
+  private static final Urn TEST_CONTAINER_URN =
+      Urn.createFromTuple(Constants.CONTAINER_ENTITY_NAME, "container1");
+
+  @Test
+  public void testMapDataFlowContainer() throws URISyntaxException {
+    com.linkedin.container.Container input = new com.linkedin.container.Container();
+    input.setContainer(TEST_CONTAINER_URN);
+
+    final Map<String, EnvelopedAspect> containerAspect = new HashMap<>();
+    containerAspect.put(
+        Constants.CONTAINER_ASPECT_NAME,
+        new com.linkedin.entity.EnvelopedAspect().setValue(new Aspect(input.data())));
+    final EntityResponse response =
+        new EntityResponse()
+            .setEntityName(Constants.DATA_FLOW_ENTITY_NAME)
+            .setUrn(TEST_DATA_FLOW_URN)
+            .setAspects(new EnvelopedAspectMap(containerAspect));
+
+    final DataFlow actual = DataFlowMapper.map(null, response);
+
+    Assert.assertEquals(actual.getUrn(), TEST_DATA_FLOW_URN.toString());
+    Assert.assertEquals(actual.getContainer().getUrn(), TEST_CONTAINER_URN.toString());
+  }
+}
diff --git a/...e/src/test/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapperTest.java b/...e/src/test/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapperTest.java
@@ -0,0 +1,42 @@
+package com.linkedin.datahub.graphql.types.datajob.mappers;
+
+import com.linkedin.common.urn.Urn;
+import com.linkedin.datahub.graphql.generated.DataJob;
+import com.linkedin.entity.Aspect;
+import com.linkedin.entity.EntityResponse;
+import com.linkedin.entity.EnvelopedAspect;
+import com.linkedin.entity.EnvelopedAspectMap;
+import com.linkedin.metadata.Constants;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class DataJobMapperTest {
+  private static final Urn TEST_DATA_JOB_URN =
+      Urn.createFromTuple(Constants.DATA_JOB_ENTITY_NAME, "datajob1");
+  private static final Urn TEST_CONTAINER_URN =
+      Urn.createFromTuple(Constants.CONTAINER_ENTITY_NAME, "container1");
+
+  @Test
+  public void testMapDataJobContainer() throws URISyntaxException {
+    com.linkedin.container.Container input = new com.linkedin.container.Container();
+    input.setContainer(TEST_CONTAINER_URN);
+
+    final Map<String, EnvelopedAspect> containerAspect = new HashMap<>();
+    containerAspect.put(
+        Constants.CONTAINER_ASPECT_NAME,
+        new com.linkedin.entity.EnvelopedAspect().setValue(new Aspect(input.data())));
+    final EntityResponse response =
+        new EntityResponse()
+            .setEntityName(Constants.DATA_JOB_ENTITY_NAME)
+            .setUrn(TEST_DATA_JOB_URN)
+            .setAspects(new EnvelopedAspectMap(containerAspect));
+
+    final DataJob actual = DataJobMapper.map(null, response);
+
+    Assert.assertEquals(actual.getUrn(), TEST_DATA_JOB_URN.toString());
+    Assert.assertEquals(actual.getContainer().getUrn(), TEST_CONTAINER_URN.toString());
+  }
+}
diff --git a/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx b/datahub-web-react/src/app/entity/dataFlow/DataFlowEntity.tsx
@@ -184,6 +184,7 @@ export class DataFlowEntity implements Entity<DataFlow> {
                 degree={(result as any).degree}
                 paths={(result as any).paths}
                 health={data.health}
+                parentContainers={data.parentContainers}
             />
         );
     };

diff --git a/datahub-web-react/src/app/entity/dataFlow/preview/Preview.tsx b/datahub-web-react/src/app/entity/dataFlow/preview/Preview.tsx
@@ -10,6 +10,7 @@ import {
     GlobalTags,
     Health,
     Owner,
+    ParentContainersResult,
     SearchInsight,
 } from '../../../../types.generated';
 import DefaultPreviewCard from '../../../preview/DefaultPreviewCard';
@@ -40,6 +41,7 @@ export const Preview = ({
     degree,
     paths,
     health,
+    parentContainers,
 }: {
     urn: string;
     name: string;
@@ -59,6 +61,7 @@ export const Preview = ({
     degree?: number;
     paths?: EntityPath[];
     health?: Health[] | null;
+    parentContainers?: ParentContainersResult | null;
 }): JSX.Element => {
     const entityRegistry = useEntityRegistry();
     return (
@@ -91,6 +94,7 @@ export const Preview = ({
             degree={degree}
             paths={paths}
             health={health || undefined}
+            parentContainers={parentContainers}
         />
     );
 };
diff --git a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx
@@ -205,6 +205,7 @@ export class DataJobEntity implements Entity<DataJob> {
                 degree={(result as any).degree}
                 paths={(result as any).paths}
                 health={data.health}
+                parentContainers={data.parentContainers}
             />
         );
     };
-Original file line number
+Diff line change
@@ Expand Up @@
         resolutionStrategy.force externalDependency.antlr4Runtime
         resolutionStrategy.force externalDependency.antlr4
+        resolutionStrategy.force 'org.apache.mina:mina-core:2.2.4'
       }
     }
@@ Expand Down @@