Merge branch 'master' into pb-remove-0

datahub-project · Aug 9, 2024 · 8ac124f · 8ac124f
2 parents 5691990 + b1f16f9
commit 8ac124f
Show file tree

Hide file tree

Showing 1,109 changed files with 221,813 additions and 15,748 deletions.
diff --git a/.github/scripts/check_python_package.py b/.github/scripts/check_python_package.py
@@ -1,18 +1,33 @@
 import setuptools
+import os
 
 folders = ["./smoke-test/tests"]
 
 for folder in folders:
     print(f"Checking folder {folder}")
-    a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
-    b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
+    packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
+    namespace_packages = [
+        i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i
+    ]
 
-    in_a_not_b = set(a) - set(b)
-    in_b_not_a = set(b) - set(a)
+    print("Packages found:", packages)
+    print("Namespace packages found:", namespace_packages)
+
+    in_packages_not_namespace = set(packages) - set(namespace_packages)
+    in_namespace_not_packages = set(namespace_packages) - set(packages)
+
+    if in_packages_not_namespace:
+        print(f"Packages not in namespace packages: {in_packages_not_namespace}")
+    if in_namespace_not_packages:
+        print(f"Namespace packages not in packages: {in_namespace_not_packages}")
+        for pkg in in_namespace_not_packages:
+            pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep))
+            print(f"Contents of {pkg_path}:")
+            print(os.listdir(pkg_path))
 
     assert (
-        len(in_a_not_b) == 0
-    ), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
+        len(in_packages_not_namespace) == 0
+    ), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}"
     assert (
-        len(in_b_not_a) == 0
-    ), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"
+        len(in_namespace_not_packages) == 0
+    ), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}"
diff --git a/.github/scripts/docker_helpers.sh b/.github/scripts/docker_helpers.sh
@@ -16,11 +16,11 @@ function get_tag {
 }
 
 function get_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
 }
 
 function get_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
 }
 
 function get_python_docker_release_v {
@@ -32,9 +32,9 @@ function get_unique_tag {
 }
 
 function get_unique_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
 }
 
 function get_unique_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
 }
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -40,19 +40,19 @@ jobs:
             extra_pip_requirements: "apache-airflow~=2.2.4"
             extra_pip_extras: plugin-v1
           - python-version: "3.10"
-            extra_pip_requirements: "apache-airflow==2.4.3"
+            extra_pip_requirements: "apache-airflow~=2.4.3"
             extra_pip_extras: plugin-v2,test-airflow24
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
+            extra_pip_requirements: 'apache-airflow~=2.6.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
+            extra_pip_requirements: 'apache-airflow~=2.7.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
           - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
+            extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
             extra_pip_extras: plugin-v2
-          - python-version: "3.10"
-            extra_pip_requirements: 'apache-airflow==2.9.0 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.10.txt'
+          - python-version: "3.11"
+            extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
             extra_pip_extras: plugin-v2
       fail-fast: false
     steps:
@@ -61,7 +61,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -67,7 +67,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -33,7 +33,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -126,7 +126,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Run lint on smoke test
         if: ${{ steps.ci-optimize.outputs.smoke-test-change == 'true' }}
         run: |
@@ -145,7 +145,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -209,7 +209,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -273,7 +273,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -337,7 +337,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -401,7 +401,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Pre-build artifacts for docker image
@@ -643,7 +643,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build codegen
@@ -730,7 +730,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
       - name: Build codegen
@@ -857,7 +857,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -27,6 +27,8 @@ concurrency:
 jobs:
   gh-pages:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       # We explicitly don't use acryldata/sane-checkout-action because docusaurus runs
       # git commands to determine the last change date for each file.
@@ -38,7 +40,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -51,7 +51,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
@@ -53,7 +53,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
         with:

diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
@@ -34,7 +34,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: acryldata/sane-checkout-action@v3
       - uses: actions/setup-python@v4
         with:

diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -30,22 +30,21 @@ jobs:
                 "treff7es",
                 "yoonhyejin",
                 "eboneil",
-                "ethan-cartwright",
                 "gabe-lyons",
                 "hsheth2",
                 "jjoyce0510",
                 "maggiehays",
-                "mrjefflewis",
                 "pedro93",
                 "RyanHolstien",
                 "Kunal-kankriya",
                 "purnimagarg1",
-                "gaurav2733",
                 "dushayntAW",
-                "AvaniSiddhapuraAPT",
-                "akarsh991",
-                "shubhamjagtap639",
-                "mayurinehate"
+                "sagar-salvi-apptware",
+                "kushagra-apptware",
+                "Salman-Apptware",
+                "mayurinehate",
+                "noggi",
+                "skrydal"
               ]'), 
               github.actor
             ) 
@@ -60,7 +59,6 @@ jobs:
           ${{ 
             contains(
               fromJson('[
-                "skrydal",
                 "siladitya2",
                 "sgomezvillamor",
                 "ngamanda",

diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
@@ -54,7 +54,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/spark-smoke-test.yml b/.github/workflows/spark-smoke-test.yml
@@ -35,7 +35,7 @@ jobs:
         with:
           distribution: "zulu"
           java-version: 17
-      - uses: gradle/gradle-build-action@v2
+      - uses: gradle/actions/setup-gradle@v3
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/README.md b/README.md
@@ -118,6 +118,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Cabify](https://cabify.tech/)
 - [ClassDojo](https://www.classdojo.com/)
 - [Coursera](https://www.coursera.org/)
+- [CVS Health](https://www.cvshealth.com/)
 - [DefinedCrowd](http://www.definedcrowd.com)
 - [DFDS](https://www.dfds.com/)
 - [Digital Turbine](https://www.digitalturbine.com/)
@@ -137,7 +138,6 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Peloton](https://www.onepeloton.com)
 - [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/)
 - [Razer](https://www.razer.com)
-- [Saxo Bank](https://www.home.saxo)
 - [Showroomprive](https://www.showroomprive.com/)
 - [SpotHero](https://spothero.com)
 - [Stash](https://www.stash.com)

diff --git a/build.gradle b/build.gradle
@@ -54,7 +54,7 @@ buildscript {
   ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '12.16.1'
   ext.googleJavaFormatVersion = '1.18.1'
-  ext.openLineageVersion = '1.16.0'
+  ext.openLineageVersion = '1.19.0'
   ext.logbackClassicJava8 = '1.2.12'
 
   ext.docker_registry = 'acryldata'
@@ -111,6 +111,7 @@ project.ext.externalDependency = [
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
     'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
     'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:2.0.3',
+    'awsS3': 'software.amazon.awssdk:s3:2.26.21',
     'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
     'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
     'awsRds':'software.amazon.awssdk:rds:2.18.24',

diff --git a/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java b/datahub-frontend/app/auth/sso/oidc/OidcConfigs.java
@@ -41,6 +41,8 @@ public class OidcConfigs extends SsoConfigs {
   public static final String OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS =
       "auth.oidc.extractJwtAccessTokenClaims";
   public static final String OIDC_PREFERRED_JWS_ALGORITHM = "auth.oidc.preferredJwsAlgorithm";
+  public static final String OIDC_GRANT_TYPE = "auth.oidc.grantType";
+  public static final String OIDC_ACR_VALUES = "auth.oidc.acrValues";
 
   /** Default values */
   private static final String DEFAULT_OIDC_USERNAME_CLAIM = "email";
@@ -75,7 +77,9 @@ public class OidcConfigs extends SsoConfigs {
   private final Optional<String> customParamResource;
   private final String readTimeout;
   private final Optional<Boolean> extractJwtAccessTokenClaims;
-  private Optional<String> preferredJwsAlgorithm;
+  private final Optional<String> preferredJwsAlgorithm;
+  private final Optional<String> grantType;
+  private final Optional<String> acrValues;
 
   public OidcConfigs(Builder builder) {
     super(builder);
@@ -98,6 +102,8 @@ public OidcConfigs(Builder builder) {
     this.readTimeout = builder.readTimeout;
     this.extractJwtAccessTokenClaims = builder.extractJwtAccessTokenClaims;
     this.preferredJwsAlgorithm = builder.preferredJwsAlgorithm;
+    this.acrValues = builder.acrValues;
+    this.grantType = builder.grantType;
   }
 
   public static class Builder extends SsoConfigs.Builder<Builder> {
@@ -123,6 +129,8 @@ public static class Builder extends SsoConfigs.Builder<Builder> {
     private String readTimeout = DEFAULT_OIDC_READ_TIMEOUT;
     private Optional<Boolean> extractJwtAccessTokenClaims = Optional.empty();
     private Optional<String> preferredJwsAlgorithm = Optional.empty();
+    private Optional<String> grantType = Optional.empty();
+    private Optional<String> acrValues = Optional.empty();
 
     public Builder from(final com.typesafe.config.Config configs) {
       super.from(configs);
@@ -169,6 +177,8 @@ public Builder from(final com.typesafe.config.Config configs) {
           getOptional(configs, OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).map(Boolean::parseBoolean);
       preferredJwsAlgorithm =
           Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));
+      grantType = Optional.ofNullable(getOptional(configs, OIDC_GRANT_TYPE, null));
+      acrValues = Optional.ofNullable(getOptional(configs, OIDC_ACR_VALUES, null));
       return this;
     }