Merge branch 'master' into snowflake-streams-v2

build.gradle

@@ -193,7 +193,7 @@ project.ext.externalDependency = [
     'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion",
     // avro-serde includes dependencies for `kafka-avro-serializer` `kafka-schema-registry-client` and `avro`
     'kafkaAvroSerde': "io.confluent:kafka-streams-avro-serde:$kafkaVersion",
-    'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
+    'kafkaAvroSerializer': "io.confluent:kafka-avro-serializer:$kafkaVersion",
     'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion-ccs",
     'snappy': 'org.xerial.snappy:snappy-java:1.1.10.5',
     'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java

@@ -232,6 +232,10 @@ public static <T> T restrictEntity(@Nonnull Object entity, Class<T> clazz) {
     try {
       Object[] args =
           allFields.stream()
+              // New versions of graphql.codegen generate serialVersionUID
+              // We need to filter serialVersionUID out because serialVersionUID is
+              // never part of the entity type constructor
+              .filter(field -> !field.getName().contains("serialVersionUID"))
               .map(
                   field -> {
                     // properties are often not required but only because

datahub-web-react/src/app/context/CustomUserContext.tsx

@@ -0,0 +1,7 @@
+/**
+ * Custom User Context State - This is a custom user context state and can be overriden in specific fork of DataHub.
+ * The below type can be customized with specific object properties as well if needed.
+ */
+export type CustomUserContextState = Record<string, any>;
+
+export const DEFAULT_CUSTOM_STATE: CustomUserContextState = {};

datahub-web-react/src/app/context/userContext.tsx

@@ -1,5 +1,6 @@
 import React from 'react';
 import { CorpUser, PlatformPrivileges } from '../../types.generated';
+import { CustomUserContextState, DEFAULT_CUSTOM_STATE } from './CustomUserContext';
 
 /**
  * Local State is persisted to local storage.
@@ -22,6 +23,7 @@ export type State = {
         loadedPersonalDefaultViewUrn: boolean;
         hasSetDefaultView: boolean;
     };
+    customState?: CustomUserContextState;
 };
 
 /**
@@ -51,6 +53,7 @@ export const DEFAULT_STATE: State = {
         loadedPersonalDefaultViewUrn: false,
         hasSetDefaultView: false,
     },
+    customState: DEFAULT_CUSTOM_STATE,
 };
 
 export const DEFAULT_CONTEXT = {

datahub-web-react/src/app/entity/shared/containers/profile/header/EntityName.tsx

@@ -48,9 +48,9 @@ function EntityName(props: Props) {
             setIsEditing(false);
             return;
         }
-        setUpdatedName(name);
         updateName({ variables: { input: { name, urn } } })
             .then(() => {
+                setUpdatedName(name);
                 setIsEditing(false);
                 message.success({ content: 'Name Updated', duration: 2 });
                 refetch();

datahub-web-react/src/app/entity/shared/containers/profile/sidebar/Ownership/EditOwnersModal.tsx

@@ -78,10 +78,26 @@ export const EditOwnersModal = ({
     const renderSearchResult = (entity: Entity) => {
         const avatarUrl =
             (entity.type === EntityType.CorpUser && (entity as CorpUser).editableProperties?.pictureLink) || undefined;
+        const corpUserDepartmentName =
+            (entity.type === EntityType.CorpUser && (entity as CorpUser).properties?.departmentName) || '';
+        const corpUserId = (entity.type === EntityType.CorpUser && (entity as CorpUser).username) || '';
+        const corpUserTitle = (entity.type === EntityType.CorpUser && (entity as CorpUser).properties?.title) || '';
         const displayName = entityRegistry.getDisplayName(entity.type, entity);
+
         return (
-            <Select.Option value={entity.urn} key={entity.urn}>
-                <OwnerLabel name={displayName} avatarUrl={avatarUrl} type={entity.type} />
+            <Select.Option
+                key={entity.urn}
+                value={entity.urn}
+                label={<OwnerLabel name={displayName} avatarUrl={avatarUrl} type={entity.type} />}
+            >
+                <OwnerLabel
+                    name={displayName}
+                    avatarUrl={avatarUrl}
+                    type={entity.type}
+                    corpUserId={corpUserId}
+                    corpUserTitle={corpUserTitle}
+                    corpUserDepartmentName={corpUserDepartmentName}
+                />
             </Select.Option>
         );
     };
@@ -381,6 +397,7 @@ export const EditOwnersModal = ({
                                 value: owner.value.ownerUrn,
                                 label: owner.label,
                             }))}
+                            optionLabelProp="label"
                         >
                             {ownerSearchOptions}
                         </SelectInput>

datahub-web-react/src/app/shared/OwnerLabel.tsx

@@ -20,14 +20,22 @@ type Props = {
     name: string;
     avatarUrl: string | undefined;
     type: EntityType;
+    corpUserId?: string;
+    corpUserTitle?: string;
+    corpUserDepartmentName?: string;
 };
 
-export const OwnerLabel = ({ name, avatarUrl, type }: Props) => {
+export const OwnerLabel = ({ name, avatarUrl, type, corpUserId, corpUserTitle, corpUserDepartmentName }: Props) => {
+    const subHeader = [corpUserId, corpUserTitle, corpUserDepartmentName].filter(Boolean).join(' - ');
+
     return (
         <OwnerContainerWrapper>
             <OwnerContentWrapper>
                 <CustomAvatar size={24} name={name} photoUrl={avatarUrl} isGroup={type === EntityType.CorpGroup} />
-                <div>{name}</div>
+                <div>
+                    <div>{name}</div>
+                    {subHeader && <div style={{ color: 'gray' }}>{subHeader}</div>}
+                </div>
             </OwnerContentWrapper>
         </OwnerContainerWrapper>
     );

datahub-web-react/src/graphql/search.graphql

@@ -433,6 +433,8 @@ fragment searchResultsWithoutSchemaField on Entity {
             lastName
             fullName
             email
+            departmentName
+            title
         }
         info {
             active
@@ -442,6 +444,8 @@ fragment searchResultsWithoutSchemaField on Entity {
             lastName
             fullName
             email
+            departmentName
+            title
         }
         editableProperties {
             displayName

docs-website/docusaurus.config.js

@@ -77,7 +77,7 @@ module.exports = {
     announcementBar: {
           id: "announcement-3",
           content:
-            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><!--div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div--><p>Watch Metadata & AI Summit sessions on-demand.</p><a href="https://www.youtube.com/@DataHubProject/videos" target="_blank" class="button">Watch Now<span> →</span></a></div>',
+            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><!--div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div--><p>Learn about DataHub 1.0 launching at our 5th birthday party!</p><a href="https://lu.ma/0j5jcocn" target="_blank" class="button">Register<span> →</span></a></div>',
           backgroundColor: "#111",
           textColor: "#ffffff",
           isCloseable: false,

docs-website/src/components/SolutionsDropdown/SolutionsDropdownContent/solutionsDropdownContent.js

@@ -24,7 +24,7 @@ const solutionsDropdownContent = {
             title: "DataHub Core",
             description: "Get started with the Open Source platform.",
             iconImage: "/img/solutions/icon-dropdown-core.png",
-            href: "/",
+            href: "/docs/quickstart",
         },
         {
             title: "Cloud vs Core",

docs/cli.md

@@ -115,17 +115,17 @@ datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --dry-run
 datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n
 ```
 
-#### ingest --list-source-runs
+#### ingest list-source-runs
 
-The `--list-source-runs` option of the `ingest` command lists the previous runs, displaying their run ID, source name, 
+The `list-source-runs` option of the `ingest` command lists the previous runs, displaying their run ID, source name, 
 start time, status, and source URN. This command allows you to filter results using the --urn option for URN-based 
 filtering or the --source option to filter by source name (partial or complete matches are supported).
 
 ```shell
 # List all ingestion runs
-datahub ingest --list-source-runs
+datahub ingest list-source-runs
 # Filter runs by a source name containing "demo"
-datahub ingest --list-source-runs --source "demo"
+datahub ingest list-source-runs --source "demo"
 ```
 
 #### ingest --preview

li-utils/src/main/javaPegasus/com/linkedin/common/urn/DataPlatformInstanceUrn.java

@@ -0,0 +1,79 @@
+package com.linkedin.common.urn;
+
+import com.linkedin.data.template.Custom;
+import com.linkedin.data.template.DirectCoercer;
+import com.linkedin.data.template.TemplateOutputCastException;
+import java.net.URISyntaxException;
+
+public final class DataPlatformInstanceUrn extends Urn {
+
+  public static final String ENTITY_TYPE = "dataPlatformInstance";
+
+  private final DataPlatformUrn _platform;
+  private final String _instanceId;
+
+  public DataPlatformInstanceUrn(DataPlatformUrn platform, String instanceId) {
+    super(ENTITY_TYPE, TupleKey.create(platform, instanceId));
+    this._platform = platform;
+    this._instanceId = instanceId;
+  }
+
+  public DataPlatformUrn getPlatformEntity() {
+    return _platform;
+  }
+
+  public String getInstance() {
+    return _instanceId;
+  }
+
+  public static DataPlatformInstanceUrn createFromString(String rawUrn) throws URISyntaxException {
+    return createFromUrn(Urn.createFromString(rawUrn));
+  }
+
+  public static DataPlatformInstanceUrn createFromUrn(Urn urn) throws URISyntaxException {
+    if (!"li".equals(urn.getNamespace())) {
+      throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'.");
+    } else if (!ENTITY_TYPE.equals(urn.getEntityType())) {
+      throw new URISyntaxException(
+          urn.toString(), "Urn entity type should be 'dataPlatformInstance'.");
+    } else {
+      TupleKey key = urn.getEntityKey();
+      if (key.size() != 2) {
+        throw new URISyntaxException(urn.toString(), "Invalid number of keys.");
+      } else {
+        try {
+          return new DataPlatformInstanceUrn(
+              (DataPlatformUrn) key.getAs(0, DataPlatformUrn.class),
+              (String) key.getAs(1, String.class));
+        } catch (Exception e) {
+          throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + e.getMessage());
+        }
+      }
+    }
+  }
+
+  public static DataPlatformInstanceUrn deserialize(String rawUrn) throws URISyntaxException {
+    return createFromString(rawUrn);
+  }
+
+  static {
+    Custom.initializeCustomClass(DataPlatformUrn.class);
+    Custom.initializeCustomClass(DataPlatformInstanceUrn.class);
+    Custom.registerCoercer(
+        new DirectCoercer<DataPlatformInstanceUrn>() {
+          public Object coerceInput(DataPlatformInstanceUrn object) throws ClassCastException {
+            return object.toString();
+          }
+
+          public DataPlatformInstanceUrn coerceOutput(Object object)
+              throws TemplateOutputCastException {
+            try {
+              return DataPlatformInstanceUrn.createFromString((String) object);
+            } catch (URISyntaxException e) {
+              throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e);
+            }
+          }
+        },
+        DataPlatformInstanceUrn.class);
+  }
+}

li-utils/src/main/pegasus/com/linkedin/common/DataPlatformInstanceUrn.pdl

@@ -0,0 +1,27 @@
+namespace com.linkedin.common
+
+/**
+ * Standardized dataset identifier.
+ */
+@java.class = "com.linkedin.common.urn.DataPlatformInstanceUrn"
+@validate.`com.linkedin.common.validator.TypedUrnValidator` = {
+  "accessible" : true,
+  "owningTeam" : "urn:li:internalTeam:datahub",
+  "entityType" : "dataPlatformInstance",
+  "constructable" : true,
+  "namespace" : "li",
+  "name" : "DataPlatformInstance",
+  "doc" : "Standardized data platform instance identifier.",
+  "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ],
+  "fields" : [ {
+    "type" : "com.linkedin.common.urn.DataPlatformUrn",
+    "name" : "platform",
+    "doc" : "Standardized platform urn."
+  }, {
+    "name" : "instance",
+    "doc" : "Instance of the data platform (e.g. db instance)",
+    "type" : "string",
+  } ],
+  "maxLength" : 100
+}
+typeref DataPlatformInstanceUrn = string

metadata-ingestion/src/datahub/cli/ingest_cli.py

@@ -507,15 +507,11 @@ def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) ->
         click.echo("No response received from the server.")
         return
 
-    # when urn or source filter does not match, exit gracefully
-    if (
-        not isinstance(data.get("data"), dict)
-        or "listIngestionSources" not in data["data"]
-    ):
-        click.echo("No matching ingestion sources found. Please check your filters.")
-        return
+    # a lot of responses can be null if there's errors in the run
+    ingestion_sources = (
+        data.get("data", {}).get("listIngestionSources", {}).get("ingestionSources", [])
+    )
 
-    ingestion_sources = data["data"]["listIngestionSources"]["ingestionSources"]
     if not ingestion_sources:
         click.echo("No ingestion sources or executions found.")
         return
@@ -526,18 +522,32 @@ def list_source_runs(page_offset: int, page_size: int, urn: str, source: str) ->
         name = ingestion_source.get("name", "N/A")
 
         executions = ingestion_source.get("executions", {}).get("executionRequests", [])
+
         for execution in executions:
+            if execution is None:
+                continue
+
             execution_id = execution.get("id", "N/A")
-            start_time = execution.get("result", {}).get("startTimeMs", "N/A")
-            start_time = (
-                datetime.fromtimestamp(start_time / 1000).strftime("%Y-%m-%d %H:%M:%S")
-                if start_time != "N/A"
-                else "N/A"
-            )
-            status = execution.get("result", {}).get("status", "N/A")
+            result = execution.get("result") or {}
+            status = result.get("status", "N/A")
+
+            try:
+                start_time = (
+                    datetime.fromtimestamp(
+                        result.get("startTimeMs", 0) / 1000
+                    ).strftime("%Y-%m-%d %H:%M:%S")
+                    if status != "DUPLICATE" and result.get("startTimeMs") is not None
+                    else "N/A"
+                )
+            except (TypeError, ValueError):
+                start_time = "N/A"
 
             rows.append([execution_id, name, start_time, status, urn])
 
+    if not rows:
+        click.echo("No execution data found.")
+        return
+
     click.echo(
         tabulate(
             rows,

metadata-ingestion/src/datahub/secret/datahub_secrets_client.py

@@ -11,34 +11,25 @@ class DataHubSecretsClient:
     def __init__(self, graph: DataHubGraph):
         self.graph = graph
 
+    def _cleanup_secret_name(self, secret_names: List[str]) -> List[str]:
+        """Remove empty strings from the list of secret names."""
+        return [secret_name for secret_name in secret_names if secret_name]
+
     def get_secret_values(self, secret_names: List[str]) -> Dict[str, Optional[str]]:
         if len(secret_names) == 0:
             return {}
 
-        request_json = {
-            "query": """query getSecretValues($input: GetSecretValuesInput!) {\n
-                getSecretValues(input: $input) {\n
-                    name\n
-                    value\n
-                }\n
+        res_data = self.graph.execute_graphql(
+            query="""query getSecretValues($input: GetSecretValuesInput!) {
+                getSecretValues(input: $input) {
+                    name
+                    value
+                }
             }""",
-            "variables": {"input": {"secrets": secret_names}},
-        }
-        # TODO: Use graph.execute_graphql() instead.
-
-        # Fetch secrets using GraphQL API f
-        response = self.graph._session.post(
-            f"{self.graph.config.server}/api/graphql", json=request_json
+            variables={"input": {"secrets": self._cleanup_secret_name(secret_names)}},
         )
-        response.raise_for_status()
-
-        # Verify response
-        res_data = response.json()
-        if "errors" in res_data:
-            raise Exception("Failed to retrieve secrets from DataHub.")
-
         # Convert list of name, value secret pairs into a dict and return
-        secret_value_list = res_data["data"]["getSecretValues"]
+        secret_value_list = res_data["getSecretValues"]
         secret_value_dict = dict()
         for secret_value in secret_value_list:
             secret_value_dict[secret_value["name"]] = secret_value["value"]