Merge remote-tracking branch 'refs/remotes/origin/master+ing-467-uniq…

…ue-hive-metastore-urn' into master+ing-467-unique-hive-metastore-urn
mayurinehate · Jan 15, 2024 · 7a91f6c · 7a91f6c
2 parents 3d6cb5e + 988aae9
commit 7a91f6c
Show file tree

Hide file tree

Showing 128 changed files with 5,531 additions and 1,278 deletions.
diff --git a/build.gradle b/build.gradle
@@ -6,20 +6,21 @@ buildscript {
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
   ext.pegasusVersion = '29.48.4'
   ext.mavenVersion = '3.6.3'
-  ext.springVersion = '5.3.29'
-  ext.springBootVersion = '2.7.14'
+  ext.springVersion = '6.1.2'
+  ext.springBootVersion = '3.2.1'
+  ext.springKafkaVersion = '3.1.1'
   ext.openTelemetryVersion = '1.18.0'
   ext.neo4jVersion = '5.14.0'
   ext.neo4jTestVersion = '5.14.0'
   ext.neo4jApocVersion = '5.14.0'
   ext.testContainersVersion = '1.17.4'
   ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.3'
-  ext.jettyVersion = '9.4.46.v20220331'
+  ext.jettyVersion = '11.0.19'
   ext.playVersion = '2.8.18'
   ext.log4jVersion = '2.19.0'
   ext.slf4jVersion = '1.7.36'
-  ext.logbackClassic = '1.2.13'
+  ext.logbackClassic = '1.4.14'
   ext.hadoop3Version = '3.3.5'
   ext.kafkaVersion = '2.3.0'
   ext.hazelcastVersion = '5.3.6'
@@ -104,7 +105,8 @@ project.ext.externalDependency = [
     'graphqlJava': 'com.graphql-java:graphql-java:19.5',
     'graphqlJavaScalars': 'com.graphql-java:graphql-java-extended-scalars:19.1',
     'gson': 'com.google.code.gson:gson:2.8.9',
-    'guice': 'com.google.inject:guice:4.2.3',
+    'guice': 'com.google.inject:guice:7.0.0',
+    'guice4': 'com.google.inject:guice:4.2.3', // Used for frontend while still on old Play version
     'guava': 'com.google.guava:guava:32.1.2-jre',
     'h2': 'com.h2database:h2:2.2.224',
     'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
@@ -115,7 +117,7 @@ project.ext.externalDependency = [
     'hazelcastSpring':"com.hazelcast:hazelcast-spring:$hazelcastVersion",
     'hazelcastTest':"com.hazelcast:hazelcast:$hazelcastVersion:tests",
     'hibernateCore': 'org.hibernate:hibernate-core:5.2.16.Final',
-    'httpClient': 'org.apache.httpcomponents:httpclient:4.5.9',
+    'httpClient': 'org.apache.httpcomponents.client5:httpclient5:5.3',
     'httpAsyncClient': 'org.apache.httpcomponents:httpasyncclient:4.1.5',
     'iStackCommons': 'com.sun.istack:istack-commons-runtime:4.0.1',
     'jacksonJDK8': "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:$jacksonVersion",
@@ -192,15 +194,15 @@ project.ext.externalDependency = [
     'reflections': 'org.reflections:reflections:0.9.9',
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
     'rythmEngine': 'org.rythmengine:rythm-engine:1.3.0',
-    'servletApi': 'javax.servlet:javax.servlet-api:3.1.0',
+    'servletApi': 'jakarta.servlet:jakarta.servlet-api:6.0.0',
     'shiroCore': 'org.apache.shiro:shiro-core:1.11.0',
     'snakeYaml': 'org.yaml:snakeyaml:2.0',
     'sparkSql' : 'org.apache.spark:spark-sql_2.12:3.0.3',
     'sparkHive' : 'org.apache.spark:spark-hive_2.12:3.0.3',
     'springBeans': "org.springframework:spring-beans:$springVersion",
     'springContext': "org.springframework:spring-context:$springVersion",
     'springCore': "org.springframework:spring-core:$springVersion",
-    'springDocUI': 'org.springdoc:springdoc-openapi-ui:1.6.14',
+    'springDocUI': 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.3.0',
     'springJdbc': "org.springframework:spring-jdbc:$springVersion",
     'springWeb': "org.springframework:spring-web:$springVersion",
     'springWebMVC': "org.springframework:spring-webmvc:$springVersion",
@@ -211,7 +213,7 @@ project.ext.externalDependency = [
     'springBootStarterJetty': "org.springframework.boot:spring-boot-starter-jetty:$springBootVersion",
     'springBootStarterCache': "org.springframework.boot:spring-boot-starter-cache:$springBootVersion",
     'springBootStarterValidation': "org.springframework.boot:spring-boot-starter-validation:$springBootVersion",
-    'springKafka': 'org.springframework.kafka:spring-kafka:2.9.13',
+    'springKafka': "org.springframework.kafka:spring-kafka:$springKafkaVersion",
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
@@ -268,6 +270,8 @@ allprojects {
       javaCompiler = javaToolchains.compilerFor {
         languageVersion = JavaLanguageVersion.of(jdkVersion)
       }
+      // Puts parameter names into compiled class files, necessary for Spring 6
+      options.compilerArgs.add("-parameters")
     }
 
     tasks.withType(JavaExec).configureEach {

diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle
@@ -76,7 +76,7 @@ dependencies {
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok
-  runtimeOnly externalDependency.guice
+  runtimeOnly externalDependency.guice4
   runtimeOnly (externalDependency.playDocs) {
     exclude group: 'com.typesafe.akka', module: 'akka-http-core_2.12'
   }

diff --git a/...e/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java b/...e/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/SchemaFieldMapper.java
@@ -36,6 +36,7 @@ public SchemaField apply(
     }
     result.setIsPartOfKey(input.isIsPartOfKey());
     result.setIsPartitioningKey(input.isIsPartitioningKey());
+    result.setJsonProps(input.getJsonProps());
     return result;
   }
 

diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -2892,6 +2892,11 @@ type SchemaField {
     Whether the field is part of a partitioning key schema
     """
     isPartitioningKey: Boolean
+
+    """
+    For schema fields that have other properties that are not modeled explicitly, represented as a JSON string.
+    """
+    jsonProps: String
 }
 
 """

diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/MoveGlossaryEntityModal.tsx
@@ -7,6 +7,7 @@ import { useUpdateParentNodeMutation } from '../../../../graphql/glossary.genera
 import NodeParentSelect from './NodeParentSelect';
 import { useGlossaryEntityData } from '../GlossaryEntityContext';
 import { getGlossaryRootToUpdate, getParentNodeToUpdate, updateGlossarySidebar } from '../../../glossary/utils';
+import { getModalDomContainer } from '../../../../utils/focus';
 
 const StyledItem = styled(Form.Item)`
     margin-bottom: 0;
@@ -78,6 +79,7 @@ function MoveGlossaryEntityModal(props: Props) {
                     </Button>
                 </>
             }
+            getContainer={getModalDomContainer}
         >
             <Form form={form} initialValues={{}} layout="vertical">
                 <Form.Item
@@ -92,6 +94,7 @@ function MoveGlossaryEntityModal(props: Props) {
                             selectedParentUrn={selectedParentUrn}
                             setSelectedParentUrn={setSelectedParentUrn}
                             isMoving
+                            autofocus
                         />
                     </StyledItem>
                 </Form.Item>

diff --git a/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx b/datahub-web-react/src/app/entity/shared/EntityDropdown/NodeParentSelect.tsx
@@ -21,6 +21,7 @@ interface Props {
     selectedParentUrn: string;
     setSelectedParentUrn: (parent: string) => void;
     isMoving?: boolean;
+    autofocus?: boolean;
 }
 
 function NodeParentSelect(props: Props) {
@@ -65,6 +66,7 @@ function NodeParentSelect(props: Props) {
                 onClear={clearSelectedParent}
                 onFocus={() => setIsFocusedOnInput(true)}
                 dropdownStyle={isShowingGlossaryBrowser || !searchQuery ? { display: 'none' } : {}}
+                autoFocus={props.autofocus}
             >
                 {nodeSearchResults?.map((result) => (
                     <Select.Option key={result?.entity?.urn} value={result.entity.urn}>

diff --git a/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/AddLinkModal.tsx
@@ -5,6 +5,7 @@ import { useEntityData, useMutationUrn } from '../../EntityContext';
 import { useAddLinkMutation } from '../../../../../graphql/mutations.generated';
 import analytics, { EventType, EntityActionType } from '../../../../analytics';
 import { useUserContext } from '../../../../context/useUserContext';
+import { getModalDomContainer } from '../../../../../utils/focus';
 
 type AddLinkProps = {
     buttonProps?: Record<string, unknown>;
@@ -73,6 +74,7 @@ export const AddLinkModal = ({ buttonProps, refetch }: AddLinkProps) => {
                         Add
                     </Button>,
                 ]}
+                getContainer={getModalDomContainer}
             >
                 <Form form={form} name="addLinkForm" onFinish={handleAdd} layout="vertical">
                     <Form.Item

diff --git a/...eact/src/app/entity/shared/containers/profile/sidebar/DataProduct/SetDataProductModal.tsx b/...eact/src/app/entity/shared/containers/profile/sidebar/DataProduct/SetDataProductModal.tsx
@@ -10,6 +10,7 @@ import { IconStyleType } from '../../../../../Entity';
 import { tagRender } from '../tagRenderer';
 import { useBatchSetDataProductMutation } from '../../../../../../../graphql/dataProduct.generated';
 import { handleBatchError } from '../../../../utils';
+import { getModalDomContainer } from '../../../../../../../utils/focus';
 
 const OptionWrapper = styled.div`
     padding: 2px 0;
@@ -126,6 +127,7 @@ export default function SetDataProductModal({
                     </Button>
                 </>
             }
+            getContainer={getModalDomContainer}
         >
             <Select
                 autoFocus

diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Domain/SetDomainModal.tsx
@@ -15,6 +15,7 @@ import { BrowserWrapper } from '../../../../../../shared/tags/AddTagsTermsModal'
 import DomainNavigator from '../../../../../../domain/nestedDomains/domainNavigator/DomainNavigator';
 import ClickOutside from '../../../../../../shared/ClickOutside';
 import { ANTD_GRAY } from '../../../../constants';
+import { getModalDomContainer } from '../../../../../../../utils/focus';
 
 type Props = {
     urns: string[];
@@ -195,6 +196,7 @@ export const SetDomainModal = ({ urns, onCloseModal, refetch, defaultValue, onOk
                     </Button>
                 </>
             }
+            getContainer={getModalDomContainer}
         >
             <Form component={false}>
                 <Form.Item>

diff --git a/...-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/EditOwnersModal.tsx b/...-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/EditOwnersModal.tsx
@@ -20,6 +20,7 @@ import { useGetRecommendations } from '../../../../../../shared/recommendation';
 import { OwnerLabel } from '../../../../../../shared/OwnerLabel';
 import { handleBatchError } from '../../../../utils';
 import { useListOwnershipTypesQuery } from '../../../../../../../graphql/ownership.generated';
+import { getModalDomContainer } from '../../../../../../../utils/focus';
 
 const SelectInput = styled(Select)`
     width: 480px;
@@ -348,6 +349,7 @@ export const EditOwnersModal = ({
                     </Button>
                 </>
             }
+            getContainer={getModalDomContainer}
         >
             <Form layout="vertical" colon={false}>
                 <Form.Item key="owners" name="owners" label={<Typography.Text strong>Owner</Typography.Text>}>

diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx
@@ -5,6 +5,7 @@ import { DEFAULT_BUILDER_STATE, ViewBuilderState } from '../types';
 import { ViewBuilderForm } from './ViewBuilderForm';
 import ClickOutside from '../../../shared/ClickOutside';
 import { ViewBuilderMode } from './types';
+import { getModalDomContainer } from '../../../../utils/focus';
 
 const modalWidth = 700;
 const modalStyle = { top: 40 };
@@ -80,6 +81,7 @@ export const ViewBuilderModal = ({ mode, urn, initialState, onSubmit, onCancel }
                 width={modalWidth}
                 onCancel={onCancel}
                 data-testid="view-modal"
+                getContainer={getModalDomContainer}
             >
                 <ViewBuilderForm urn={urn} mode={mode} state={viewBuilderState} updateState={setViewBuilderState} />
                 <SaveButtonContainer>

diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -221,7 +221,7 @@
         "name": "fivetran",
         "displayName": "Fivetran",
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/fivetran/",
-        "recipe": "source:\n    type: fivetran\n    config:\n        # Fivetran log connector destination server configurations\n        fivetran_log_config:\n            destination_platform: snowflake\n            destination_config:\n                # Coordinates\n                account_id: snowflake_account_id\n                warehouse: warehouse_name\n                database: snowflake_db\n                log_schema: fivetran_log_schema\n\n                # Credentials\n                username: ${SNOWFLAKE_USER}\n                password: ${SNOWFLAKE_PASS}\n                role: snowflake_role\n\n        # Optional - filter for certain connector names instead of ingesting everything.\n        # connector_patterns:\n        #     allow:\n        #         - connector_name\n\n        # Optional -- This mapping is optional and only required to configure platform-instance for source\n        # A mapping of Fivetran connector id to data platform instance\n        # sources_to_platform_instance:\n        #     calendar_elected:\n        #         platform_instance: cloud_postgres_instance\n        #         env: DEV\n\n        # Optional -- This mapping is optional and only required to configure platform-instance for destination.\n        # A mapping of Fivetran destination id to data platform instance\n        # destination_to_platform_instance:\n        #     calendar_elected:\n        #         platform_instance: cloud_postgres_instance\n        #         env: DEV"
+        "recipe": "source:\n    type: fivetran\n    config:\n        # Fivetran log connector destination server configurations\n        fivetran_log_config:\n            destination_platform: snowflake\n            snowflake_destination_config:\n                # Coordinates\n                account_id: snowflake_account_id\n                warehouse: warehouse_name\n                database: snowflake_db\n                log_schema: fivetran_log_schema\n\n                # Credentials\n                username: ${SNOWFLAKE_USER}\n                password: ${SNOWFLAKE_PASS}\n                role: snowflake_role\n\n        # Optional - filter for certain connector names instead of ingesting everything.\n        # connector_patterns:\n        #     allow:\n        #         - connector_name\n\n        # Optional -- This mapping is optional and only required to configure platform-instance for source\n        # A mapping of Fivetran connector id to data platform instance\n        # sources_to_platform_instance:\n        #     calendar_elected:\n        #         platform_instance: cloud_postgres_instance\n        #         env: DEV\n\n        # Optional -- This mapping is optional and only required to configure platform-instance for destination.\n        # A mapping of Fivetran destination id to data platform instance\n        # destination_to_platform_instance:\n        #     calendar_elected:\n        #         platform_instance: cloud_postgres_instance\n        #         env: DEV"
     },
     {
         "urn": "urn:li:dataPlatform:csv-enricher",

diff --git a/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx b/datahub-web-react/src/app/shared/tags/AddTagsTermsModal.tsx
@@ -19,6 +19,7 @@ import { useGetRecommendations } from '../recommendation';
 import { FORBIDDEN_URN_CHARS_REGEX, handleBatchError } from '../../entity/shared/utils';
 import { TagTermLabel } from './TagTermLabel';
 import { ENTER_KEY_CODE } from '../constants';
+import { getModalDomContainer } from '../../../utils/focus';
 
 export enum OperationType {
     ADD,
@@ -448,6 +449,7 @@ export default function EditTagTermsModal({
                     </Button>
                 </>
             }
+            getContainer={getModalDomContainer}
         >
             <ClickOutside onClickOutside={() => setIsFocusedOnInput(false)}>
                 <TagSelect

diff --git a/datahub-web-react/src/utils/focus/index.ts b/datahub-web-react/src/utils/focus/index.ts
@@ -0,0 +1,6 @@
+// To focus out from modal popup so autofocus works
+// getContainer prop value when rendering modals.
+// The getContainer prop allows you to specify the container in which the modal should be rendered.
+// By default, modals are appended to the end of the document body, but using getContainer, you can specify a different container.
+export const getModalDomContainer = () => document.getElementById('root') as HTMLElement;
+// this can we remove once we upgrade to new antd version i.e 5.11.2 because there we have autoFocus property for the modal.
diff --git a/docker/datahub-gms/Dockerfile b/docker/datahub-gms/Dockerfile
@@ -42,9 +42,9 @@ RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
     && apk --no-cache add jattach --repository ${ALPINE_REPO_URL}/edge/community/ \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \
+    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/11.0.19/jetty-runner-11.0.19.jar --output jetty-runner.jar \
+    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/11.0.19/jetty-jmx-11.0.19.jar --output jetty-jmx.jar \
+    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/11.0.19/jetty-util-11.0.19.jar --output jetty-util.jar \
     && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
     && wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
     && cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks

diff --git a/docker/datahub-upgrade/Dockerfile b/docker/datahub-upgrade/Dockerfile
@@ -39,9 +39,9 @@ ENV JMX_VERSION=0.18.0
 RUN apk --no-cache --update-cache --available upgrade \
     && apk --no-cache add curl bash coreutils gcompat sqlite libc6-compat java-snappy \
     && apk --no-cache add openjdk17-jre-headless --repository=${ALPINE_REPO_URL}/edge/community \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/9.4.46.v20220331/jetty-runner-9.4.46.v20220331.jar --output jetty-runner.jar \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/9.4.46.v20220331/jetty-jmx-9.4.46.v20220331.jar --output jetty-jmx.jar \
-    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/9.4.46.v20220331/jetty-util-9.4.46.v20220331.jar --output jetty-util.jar \
+    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-runner/11.0.19/jetty-runner-11.0.19.jar --output jetty-runner.jar \
+    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-jmx/11.0.19/jetty-jmx-11.0.19.jar --output jetty-jmx.jar \
+    && curl -sS ${MAVEN_CENTRAL_REPO_URL}/org/eclipse/jetty/jetty-util/11.0.19/jetty-util-11.0.19.jar --output jetty-util.jar \
     && wget --no-verbose ${GITHUB_REPO_URL}/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.24.0/opentelemetry-javaagent.jar \
     && wget --no-verbose ${MAVEN_CENTRAL_REPO_URL}/io/prometheus/jmx/jmx_prometheus_javaagent/${JMX_VERSION}/jmx_prometheus_javaagent-${JMX_VERSION}.jar -O jmx_prometheus_javaagent.jar \
     && cp /usr/lib/jvm/java-17-openjdk/jre/lib/security/cacerts /tmp/kafka.client.truststore.jks

diff --git a/docs-website/vercel-setup.sh b/docs-website/vercel-setup.sh
@@ -29,3 +29,6 @@ rm "$py3"
 ln "$(which python3.10)" "$py3"
 python3 --version
 
+# Set up java version for gradle
+yum install java-17-amazon-corretto
+java --version