Merge branch 'master' into master

.github/actions/docker-custom-build-and-push/action.yml

@@ -70,11 +70,20 @@ runs:
         push: false
         cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
         cache-to: type=inline
+    - name: Single Tag
+      if: ${{ inputs.publish != 'true' }}
+      shell: bash
+      run: |
+        TAGS="""
+        ${{ steps.docker_meta.outputs.tags }}
+        """
+        echo "SINGLE_TAG=$(echo $TAGS | tr '\n' ' ' | awk -F' ' '{ print $1 }')" >> $GITHUB_OUTPUT
+      id: single_tag
     - name: Upload image locally for testing (if not publishing)
       uses: ishworkh/docker-image-artifact-upload@v1
       if: ${{ inputs.publish != 'true' }}
       with:
-        image: ${{ steps.docker_meta.outputs.tags }}
+        image: ${{ steps.single_tag.outputs.SINGLE_TAG }}
 
     # Code for building multi-platform images and pushing to Docker Hub.
     - name: Set up QEMU

.github/scripts/docker_helpers.sh

@@ -12,15 +12,15 @@ export SHORT_SHA=$(get_short_sha)
 echo "SHORT_SHA: $SHORT_SHA"
 
 function get_tag {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g'),${SHORT_SHA}
 }
 
 function get_tag_slim {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim\,${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
 }
 
 function get_tag_full {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full\,${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
 }
 
 function get_python_docker_release_v {

.github/workflows/docker-unified.yml

@@ -46,20 +46,26 @@ jobs:
       unique_full_tag: ${{ steps.tag.outputs.unique_full_tag }}
       publish: ${{ steps.publish.outputs.publish }}
       python_release_version: ${{ steps.tag.outputs.python_release_version }}
+      short_sha: ${{ steps.tag.outputs.short_sha }}
+      branch_name: ${{ steps.tag.outputs.branch_name }}
+      repository_name: ${{ steps.tag.outputs.repository_name }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - name: Compute Tag
         id: tag
         run: |
           source .github/scripts/docker_helpers.sh
+          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
           echo "tag=$(get_tag)" >> $GITHUB_OUTPUT
           echo "slim_tag=$(get_tag_slim)" >> $GITHUB_OUTPUT
           echo "full_tag=$(get_tag_full)" >> $GITHUB_OUTPUT
           echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
           echo "unique_slim_tag=$(get_unique_tag_slim)" >> $GITHUB_OUTPUT
           echo "unique_full_tag=$(get_unique_tag_full)" >> $GITHUB_OUTPUT
           echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
+          echo "branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+          echo "repository_name=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
@@ -860,3 +866,23 @@ jobs:
           job-status: ${{ job.status }}
           slack-bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
           channel: github-activities
+  deploy_datahub_head:
+    name: Deploy to Datahub HEAD
+    runs-on: ubuntu-latest
+    needs:
+      [
+        setup,
+        smoke_test
+      ]
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v1
+        if: ${{ needs.setup.outputs.publish != 'false' }}
+        with:
+          aws-access-key-id: ${{ secrets.AWS_SQS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SQS_ACCESS_KEY }}
+          aws-region: us-west-2
+      - uses: isbang/[email protected]
+        if: ${{ needs.setup.outputs.publish != 'false' }}
+        with:
+          sqs-url: ${{ secrets.DATAHUB_HEAD_SYNC_QUEUE }}
+          message: '{ "command": "git-sync", "args" : {"repoName": "${{ needs.setup.outputs.repository_name }}", "repoOrg": "${{ github.repository_owner }}", "repoBranch": "${{ needs.setup.outputs.branch_name }}", "repoShaShort": "${{ needs.setup.outputs.short_sha }}" }}'

.gitignore

@@ -32,6 +32,7 @@ venv.bak/
 dmypy.json
 MANIFEST
 *.pyc
+.python-version
 
 # Generated files
 **/bin

build.gradle

@@ -19,7 +19,7 @@ buildscript {
   ext.logbackClassic = '1.2.12'
   ext.hadoop3Version = '3.3.5'
   ext.kafkaVersion = '2.3.0'
-  ext.hazelcastVersion = '5.3.1'
+  ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '12.16.1'
 
   ext.docker_registry = 'linkedin'
@@ -53,7 +53,7 @@ project.ext.spec = [
         'pegasus' : [
             'd2' : 'com.linkedin.pegasus:d2:' + pegasusVersion,
             'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
-            'dataAvro1_6' : 'com.linkedin.pegasus:data-avro-1_6:' + pegasusVersion,
+            'dataAvro': 'com.linkedin.pegasus:data-avro:' + pegasusVersion,
             'generator': 'com.linkedin.pegasus:generator:' + pegasusVersion,
             'restliCommon' : 'com.linkedin.pegasus:restli-common:' + pegasusVersion,
             'restliClient' : 'com.linkedin.pegasus:restli-client:' + pegasusVersion,
@@ -71,22 +71,22 @@ project.ext.externalDependency = [
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.3',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
-    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.10',
-    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.1',
-    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.8',
-    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.0',
+    'awsGlueSchemaRegistrySerde': 'software.amazon.glue:schema-registry-serde:1.1.17',
+    'awsMskIamAuth': 'software.amazon.msk:aws-msk-iam-auth:1.1.9',
+    'awsSecretsManagerJdbc': 'com.amazonaws.secretsmanager:aws-secretsmanager-jdbc:1.0.13',
+    'awsPostgresIamAuth': 'software.amazon.jdbc:aws-advanced-jdbc-wrapper:1.0.2',
     'awsRds':'software.amazon.awssdk:rds:2.18.24',
-    'cacheApi' : 'javax.cache:cache-api:1.1.0',
+    'cacheApi': 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
     'commonsIo': 'commons-io:commons-io:2.4',
     'commonsLang': 'commons-lang:commons-lang:2.6',
     'commonsText': 'org.apache.commons:commons-text:1.10.0',
     'commonsCollections': 'commons-collections:commons-collections:3.2.2',
-    'data' : 'com.linkedin.pegasus:data:' + pegasusVersion,
+    'caffeine': 'com.github.ben-manes.caffeine:caffeine:3.1.8',
     'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
     'datastaxOssCore': 'com.datastax.oss:java-driver-core:4.14.1',
     'datastaxOssQueryBuilder': 'com.datastax.oss:java-driver-query-builder:4.14.1',
-    'dgraph4j' : 'io.dgraph:dgraph4j:21.03.1',
+    'dgraph4j' : 'io.dgraph:dgraph4j:21.12.0',
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
     'ebean': 'io.ebean:ebean:' + ebeanVersion,
@@ -131,7 +131,7 @@ project.ext.externalDependency = [
     'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.9',
-    'json': 'org.json:json:20230227',
+    'json': 'org.json:json:20231013',
     'junit': 'junit:junit:4.13.2',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
@@ -140,7 +140,7 @@ project.ext.externalDependency = [
     'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
     'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
     'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion",
-    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.3',
+    'snappy': 'org.xerial.snappy:snappy-java:1.1.10.4',
     'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
     'slf4jApi': "org.slf4j:slf4j-api:$slf4jVersion",
     'log4jCore': "org.apache.logging.log4j:log4j-core:$log4jVersion",
@@ -164,6 +164,7 @@ project.ext.externalDependency = [
     'opentelemetryAnnotations': 'io.opentelemetry:opentelemetry-extension-annotations:' + openTelemetryVersion,
     'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15',
     'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
+    'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
     'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
@@ -178,6 +179,7 @@ project.ext.externalDependency = [
     'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
     'postgresql': 'org.postgresql:postgresql:42.3.8',
     'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
+    'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
     'reflections': 'org.reflections:reflections:0.9.9',
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
@@ -201,7 +203,7 @@ project.ext.externalDependency = [
     'springBootStarterJetty': "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion",
     'springBootStarterCache': "org.springframework.boot:spring-boot-starter-cache:$springBootVersion",
     'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
-    'springKafka': 'org.springframework.kafka:spring-kafka:2.8.11',
+    'springKafka': 'org.springframework.kafka:spring-kafka:2.9.13',
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
@@ -263,7 +265,7 @@ subprojects {
   plugins.withType(JavaPlugin) {
     dependencies {
       constraints {
-        implementation('io.netty:netty-all:4.1.86.Final')
+        implementation('io.netty:netty-all:4.1.100.Final')
         implementation('org.apache.commons:commons-compress:1.21')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')

datahub-frontend/app/auth/NativeAuthenticationConfigs.java

@@ -6,18 +6,26 @@
 public class NativeAuthenticationConfigs {
 
   public static final String NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH = "auth.native.enabled";
+  public static final String NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH = "auth.native.signUp.enforceValidEmail";
 
   private Boolean _isEnabled = true;
+  private Boolean _isEnforceValidEmailEnabled = true;
 
   public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) {
-    if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH)
-        && Boolean.FALSE.equals(
-        Boolean.parseBoolean(configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString()))) {
-      _isEnabled = false;
+    if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH)) {
+      _isEnabled = Boolean.parseBoolean(configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString());
+    }
+    if (configs.hasPath(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH)) {
+      _isEnforceValidEmailEnabled =
+          Boolean.parseBoolean(configs.getValue(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH).toString());
     }
   }
 
   public boolean isNativeAuthenticationEnabled() {
     return _isEnabled;
   }
+
+  public boolean isEnforceValidEmailEnabled() {
+    return _isEnforceValidEmailEnabled;
+  }
 }

datahub-frontend/app/controllers/AuthenticationController.java

@@ -27,6 +27,7 @@
 import org.pac4j.play.store.PlaySessionStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import play.data.validation.Constraints;
 import play.libs.Json;
 import play.mvc.Controller;
 import play.mvc.Http;
@@ -203,6 +204,13 @@ public Result signUp(Http.Request request) {
             JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty.");
             return Results.badRequest(invalidCredsJson);
         }
+        if (_nativeAuthenticationConfigs.isEnforceValidEmailEnabled()) {
+            Constraints.EmailValidator emailValidator = new Constraints.EmailValidator();
+            if (!emailValidator.isValid(email)) {
+                JsonNode invalidCredsJson = Json.newObject().put("message", "Email must not be empty.");
+                return Results.badRequest(invalidCredsJson);
+            }
+        }
 
         if (StringUtils.isBlank(password)) {
             JsonNode invalidCredsJson = Json.newObject().put("message", "Password must not be empty.");

datahub-frontend/conf/application.conf

@@ -196,6 +196,10 @@ auth.oidc.preferredJwsAlgorithm = ${?AUTH_OIDC_PREFERRED_JWS_ALGORITHM} # Which
 #
 auth.jaas.enabled = ${?AUTH_JAAS_ENABLED}
 auth.native.enabled = ${?AUTH_NATIVE_ENABLED}
+
+# Enforces the usage of a valid email for user sign up
+auth.native.signUp.enforceValidEmail = true
+auth.native.signUp.enforceValidEmail = ${?ENFORCE_VALID_EMAIL}
 #
 # To disable all authentication to the app, and proxy all users through a master "datahub" account, make sure that,
 # jaas, native and oidc auth are disabled:

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java

@@ -79,11 +79,11 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
     }
 
     // 2. Get list of entities that we should query based on filters or assets from aspect.
-    List<String> entitiesToQuery = assetUrns.stream().map(Urn::getEntityType).collect(Collectors.toList());
+    List<String> entitiesToQuery = assetUrns.stream().map(Urn::getEntityType).distinct().collect(Collectors.toList());
 
 
     final List<EntityType> inputEntityTypes = (input.getTypes() == null || input.getTypes().isEmpty()) ? ImmutableList.of() : input.getTypes();
-    final List<String> inputEntityNames = inputEntityTypes.stream().map(EntityTypeMapper::getName).collect(Collectors.toList());
+    final List<String> inputEntityNames = inputEntityTypes.stream().map(EntityTypeMapper::getName).distinct().collect(Collectors.toList());
 
     final List<String> finalEntityNames = inputEntityNames.size() > 0 ? inputEntityNames : entitiesToQuery;
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java

@@ -70,7 +70,7 @@ public static List<MatchedField> getMatchedFieldEntry(List<com.linkedin.metadata
                   Urn urn = Urn.createFromString(field.getValue());
                   matchedField.setEntity(UrnToEntityMapper.map(urn));
               } catch (URISyntaxException e) {
-                  log.warn("Failed to create urn from MatchedField value: {}", field.getValue(), e);
+                  log.debug("Failed to create urn from MatchedField value: {}", field.getValue());
               }
           }
           return matchedField;

datahub-upgrade/build.gradle

@@ -66,7 +66,9 @@ dependencies {
   runtimeOnly externalDependency.mysqlConnector
   runtimeOnly externalDependency.postgresql
 
-  implementation externalDependency.awsMskIamAuth
+  implementation(externalDependency.awsMskIamAuth) {
+    exclude group: 'software.amazon.awssdk', module: 'third-party-jackson-core'
+  }
 
   annotationProcessor externalDependency.lombok
   annotationProcessor externalDependency.picocli
@@ -75,6 +77,12 @@ dependencies {
   testImplementation externalDependency.mockito
   testImplementation externalDependency.testng
   testRuntimeOnly externalDependency.logbackClassic
+
+  constraints {
+    implementation(implementation externalDependency.parquetHadoop) {
+      because("CVE-2022-42003")
+    }
+  }
 }
 
 bootJar {

datahub-web-react/.env

@@ -1,4 +1,5 @@
 PUBLIC_URL=/assets
 REACT_APP_THEME_CONFIG=theme_light.config.json
 SKIP_PREFLIGHT_CHECK=true
-BUILD_PATH=build/yarn
+BUILD_PATH=build/yarn
+REACT_APP_PROXY_TARGET=http://localhost:9002

datahub-web-react/.eslintrc.js

@@ -1,15 +1,14 @@
 module.exports = {
     parser: '@typescript-eslint/parser', // Specifies the ESLint parser
     extends: [
-        'react-app',
-        'plugin:react/recommended', // Uses the recommended rules from @eslint-plugin-react
-        'plugin:@typescript-eslint/recommended', // Uses the recommended rules from @typescript-eslint/eslint-plugin
-        'plugin:jest/recommended',
+        'airbnb',
         'airbnb-typescript',
         'airbnb/hooks',
+        'plugin:@typescript-eslint/recommended',
+        'plugin:jest/recommended',
         'prettier',
-        'plugin:prettier/recommended',
     ],
+    plugins: ['@typescript-eslint'],
     parserOptions: {
         ecmaVersion: 2020, // Allows for the parsing of modern ECMAScript features
         sourceType: 'module', // Allows for the use of imports
@@ -19,29 +18,34 @@ module.exports = {
         project: './tsconfig.json',
     },
     rules: {
-        eqeqeq: ['error', 'always'],
-        'react/destructuring-assignment': 'off',
-        'no-console': 'off',
-        'no-debugger': 'warn',
-        'require-await': 'warn',
+        '@typescript-eslint/no-explicit-any': 'off',
+        'arrow-body-style': 'off',
+        'class-methods-use-this': 'off',
+        'import/no-extraneous-dependencies': 'off',
         'import/prefer-default-export': 'off', // TODO: remove this lint rule
-        'import/extensions': 'off',
-        'react/jsx-props-no-spreading': 'off',
+        'no-console': 'off',
         'no-plusplus': 'off',
         'no-prototype-builtins': 'off',
-        'react/require-default-props': 'off',
+        'no-restricted-exports': ['off', { restrictedNamedExports: ['default', 'then'] }],
         'no-underscore-dangle': 'off',
+        'no-unsafe-optional-chaining': 'off',
+        'prefer-exponentiation-operator': 'off',
+        'prefer-regex-literals': 'off',
+        'react/destructuring-assignment': 'off',
+        'react/function-component-definition': 'off',
+        'react/jsx-no-bind': 'off',
+        'react/jsx-no-constructed-context-values': 'off',
+        'react/jsx-no-useless-fragment': 'off',
+        'react/jsx-props-no-spreading': 'off',
+        'react/no-unstable-nested-components': 'off',
+        'react/require-default-props': 'off',
         '@typescript-eslint/no-unused-vars': [
             'error',
             {
                 varsIgnorePattern: '^_',
                 argsIgnorePattern: '^_',
             },
         ],
-        '@typescript-eslint/no-empty-interface': 'off',
-        "@typescript-eslint/explicit-module-boundary-types": "off",
-        "@typescript-eslint/no-explicit-any": 'off',
-        "import/no-extraneous-dependencies": 'off'
     },
     settings: {
         react: {