Merge branch 'datahub-project:master' into master

acryldata · Nov 8, 2023 · e419a84 · e419a84
2 parents ba809ac + 353584c
commit e419a84
Show file tree

Hide file tree

Showing 34 changed files with 391 additions and 127 deletions.
diff --git a/build.gradle b/build.gradle
@@ -1,7 +1,7 @@
 buildscript {
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.45.0'
+  ext.pegasusVersion = '29.46.8'
   ext.mavenVersion = '3.6.3'
   ext.springVersion = '5.3.29'
   ext.springBootVersion = '2.7.14'
@@ -212,7 +212,7 @@ project.ext.externalDependency = [
     'testContainersOpenSearch': 'org.opensearch:opensearch-testcontainers:2.0.0',
     'typesafeConfig':'com.typesafe:config:1.4.1',
     'wiremock':'com.github.tomakehurst:wiremock:2.10.0',
-    'zookeeper': 'org.apache.zookeeper:zookeeper:3.4.14',
+    'zookeeper': 'org.apache.zookeeper:zookeeper:3.7.2',
     'wire': 'com.squareup.wire:wire-compiler:3.7.1',
     'charle':  'com.charleskorn.kaml:kaml:0.53.0',
     'common': 'commons-io:commons-io:2.7',

diff --git a/...ore/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java b/...ore/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java
@@ -40,23 +40,15 @@ public CompletableFuture<ListPoliciesResult> get(final DataFetchingEnvironment e
       final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount();
       final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery();
 
-      return CompletableFuture.supplyAsync(() -> {
-        try {
-          // First, get all policy Urns.
-          final PolicyFetcher.PolicyFetchResult policyFetchResult =
-              _policyFetcher.fetchPolicies(start, count, query, context.getAuthentication());
-
-          // Now that we have entities we can bind this to a result.
-          final ListPoliciesResult result = new ListPoliciesResult();
-          result.setStart(start);
-          result.setCount(count);
-          result.setTotal(policyFetchResult.getTotal());
-          result.setPolicies(mapEntities(policyFetchResult.getPolicies()));
-          return result;
-        } catch (Exception e) {
-          throw new RuntimeException("Failed to list policies", e);
-        }
-      });
+      return _policyFetcher.fetchPolicies(start, query, count, context.getAuthentication())
+              .thenApply(policyFetchResult -> {
+                final ListPoliciesResult result = new ListPoliciesResult();
+                result.setStart(start);
+                result.setCount(count);
+                result.setTotal(policyFetchResult.getTotal());
+                result.setPolicies(mapEntities(policyFetchResult.getPolicies()));
+                return result;
+              });
     }
     throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
   }

diff --git a/...main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java b/...main/java/com/linkedin/datahub/upgrade/system/entity/steps/BackfillBrowsePathsV2Step.java
@@ -6,13 +6,15 @@
 import com.linkedin.common.BrowsePathsV2;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
+import com.linkedin.data.template.StringArray;
 import com.linkedin.datahub.upgrade.UpgradeContext;
 import com.linkedin.datahub.upgrade.UpgradeStep;
 import com.linkedin.datahub.upgrade.UpgradeStepResult;
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
 import com.linkedin.events.metadata.ChangeType;
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
@@ -37,6 +39,8 @@
 public class BackfillBrowsePathsV2Step implements UpgradeStep {
 
   public static final String BACKFILL_BROWSE_PATHS_V2 = "BACKFILL_BROWSE_PATHS_V2";
+  public static final String REPROCESS_DEFAULT_BROWSE_PATHS_V2 = "REPROCESS_DEFAULT_BROWSE_PATHS_V2";
+  public static final String DEFAULT_BROWSE_PATH_V2 = "␟Default";
 
   private static final Set<String> ENTITY_TYPES_TO_MIGRATE = ImmutableSet.of(
       Constants.DATASET_ENTITY_NAME,
@@ -81,37 +85,24 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {
 
   private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, String scrollId) {
 
-    // Condition: has `browsePaths` AND does NOT have `browsePathV2`
-    Criterion missingBrowsePathV2 = new Criterion();
-    missingBrowsePathV2.setCondition(Condition.IS_NULL);
-    missingBrowsePathV2.setField("browsePathV2");
-    // Excludes entities without browsePaths
-    Criterion hasBrowsePathV1 = new Criterion();
-    hasBrowsePathV1.setCondition(Condition.EXISTS);
-    hasBrowsePathV1.setField("browsePaths");
-
-    CriterionArray criterionArray = new CriterionArray();
-    criterionArray.add(missingBrowsePathV2);
-    criterionArray.add(hasBrowsePathV1);
-
-    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
-    conjunctiveCriterion.setAnd(criterionArray);
+    final Filter filter;
 
-    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
-    conjunctiveCriterionArray.add(conjunctiveCriterion);
-
-    Filter filter = new Filter();
-    filter.setOr(conjunctiveCriterionArray);
+    if (System.getenv().containsKey(REPROCESS_DEFAULT_BROWSE_PATHS_V2)
+            && Boolean.parseBoolean(System.getenv(REPROCESS_DEFAULT_BROWSE_PATHS_V2))) {
+      filter = backfillDefaultBrowsePathsV2Filter();
+    } else  {
+      filter = backfillBrowsePathsV2Filter();
+    }
 
     final ScrollResult scrollResult = _searchService.scrollAcrossEntities(
         ImmutableList.of(entityType),
         "*",
         filter,
         null,
         scrollId,
-        "5m",
+        null,
         BATCH_SIZE,
-        null
+        new SearchFlags().setFulltext(true).setSkipCache(true).setSkipHighlighting(true).setSkipAggregates(true)
     );
     if (scrollResult.getNumEntities() == 0 || scrollResult.getEntities().size() == 0) {
       return null;
@@ -129,6 +120,55 @@ private String backfillBrowsePathsV2(String entityType, AuditStamp auditStamp, S
     return scrollResult.getScrollId();
   }
 
+  private Filter backfillBrowsePathsV2Filter() {
+    // Condition: has `browsePaths` AND does NOT have `browsePathV2`
+    Criterion missingBrowsePathV2 = new Criterion();
+    missingBrowsePathV2.setCondition(Condition.IS_NULL);
+    missingBrowsePathV2.setField("browsePathV2");
+    // Excludes entities without browsePaths
+    Criterion hasBrowsePathV1 = new Criterion();
+    hasBrowsePathV1.setCondition(Condition.EXISTS);
+    hasBrowsePathV1.setField("browsePaths");
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(missingBrowsePathV2);
+    criterionArray.add(hasBrowsePathV1);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+    return filter;
+  }
+
+  private Filter backfillDefaultBrowsePathsV2Filter() {
+    // Condition: has default `browsePathV2`
+    Criterion hasDefaultBrowsePathV2 = new Criterion();
+    hasDefaultBrowsePathV2.setCondition(Condition.EQUAL);
+    hasDefaultBrowsePathV2.setField("browsePathV2");
+    StringArray values = new StringArray();
+    values.add(DEFAULT_BROWSE_PATH_V2);
+    hasDefaultBrowsePathV2.setValues(values);
+    hasDefaultBrowsePathV2.setValue(DEFAULT_BROWSE_PATH_V2); // not used, but required field?
+
+    CriterionArray criterionArray = new CriterionArray();
+    criterionArray.add(hasDefaultBrowsePathV2);
+
+    ConjunctiveCriterion conjunctiveCriterion = new ConjunctiveCriterion();
+    conjunctiveCriterion.setAnd(criterionArray);
+
+    ConjunctiveCriterionArray conjunctiveCriterionArray = new ConjunctiveCriterionArray();
+    conjunctiveCriterionArray.add(conjunctiveCriterion);
+
+    Filter filter = new Filter();
+    filter.setOr(conjunctiveCriterionArray);
+    return filter;
+  }
+
   private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exception {
     BrowsePathsV2 browsePathsV2 = _entityService.buildDefaultBrowsePathV2(urn, true);
     log.debug(String.format("Adding browse path v2 for urn %s with value %s", urn, browsePathsV2));
@@ -142,7 +182,7 @@ private void ingestBrowsePathsV2(Urn urn, AuditStamp auditStamp) throws Exceptio
     _entityService.ingestProposal(
         proposal,
         auditStamp,
-        false
+        true
     );
   }
 

diff --git a/datahub-web-react/codegen.yml b/datahub-web-react/codegen.yml
@@ -20,6 +20,9 @@ generates:
     src/types.generated.ts:
         plugins:
             - 'typescript'
+    src/possibleTypes.generated.ts:
+        plugins:
+            - 'fragment-matcher'
     src/:
         preset: near-operation-file
         presetConfig:

diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
@@ -11,6 +11,7 @@
         "@apollo/client": "^3.3.19",
         "@craco/craco": "^6.1.1",
         "@data-ui/xy-chart": "^0.0.84",
+        "@graphql-codegen/fragment-matcher": "^5.0.0",
         "@miragejs/graphql": "^0.1.11",
         "@monaco-editor/react": "^4.3.1",
         "@react-hook/window-size": "^3.0.7",

diff --git a/datahub-web-react/src/App.tsx b/datahub-web-react/src/App.tsx
@@ -36,6 +36,7 @@ import { DataPlatformEntity } from './app/entity/dataPlatform/DataPlatformEntity
 import { DataProductEntity } from './app/entity/dataProduct/DataProductEntity';
 import { DataPlatformInstanceEntity } from './app/entity/dataPlatformInstance/DataPlatformInstanceEntity';
 import { RoleEntity } from './app/entity/Access/RoleEntity';
+import possibleTypesResult from './possibleTypes.generated';
 
 /*
     Construct Apollo Client
@@ -77,6 +78,8 @@ const client = new ApolloClient({
                 },
             },
         },
+        // need to define possibleTypes to allow us to use Apollo cache with union types
+        possibleTypes: possibleTypesResult.possibleTypes,
     }),
     credentials: 'include',
     defaultOptions: {

diff --git a/datahub-web-react/src/app/home/HomePageHeader.tsx b/datahub-web-react/src/app/home/HomePageHeader.tsx
@@ -276,6 +276,7 @@ export const HomePageHeader = () => {
                         combineSiblings
                         showQuickFilters
                         showViewAllResults
+                        showCommandK
                     />
                     {searchResultsToShow && searchResultsToShow.length > 0 && (
                         <SuggestionsContainer>

diff --git a/datahub-web-react/src/app/lineage/LineageExplorer.tsx b/datahub-web-react/src/app/lineage/LineageExplorer.tsx
@@ -217,7 +217,7 @@ export default function LineageExplorer({ urn, type }: Props) {
                                 Close
                             </Button>
                             <Button href={entityRegistry.getEntityUrl(selectedEntity.type, selectedEntity.urn)}>
-                                <InfoCircleOutlined /> {entityRegistry.getEntityName(selectedEntity.type)} Details
+                                <InfoCircleOutlined /> View details
                             </Button>
                         </FooterButtonGroup>
                     )

diff --git a/datahub-web-react/src/app/search/CommandK.tsx b/datahub-web-react/src/app/search/CommandK.tsx
@@ -0,0 +1,29 @@
+import React from 'react';
+import styled from 'styled-components';
+import { ANTD_GRAY } from '../entity/shared/constants';
+
+const Container = styled.div`
+    color: ${ANTD_GRAY[6]};
+    background-color: #ffffff;
+    opacity: 0.9;
+    border-color: black;
+    border-radius: 6px;
+    border: 1px solid ${ANTD_GRAY[6]};
+    padding-right: 6px;
+    padding-left: 6px;
+    margin-right: 4px;
+    margin-left: 4px;
+`;
+
+const Letter = styled.span`
+    padding: 2px;
+`;
+
+export const CommandK = () => {
+    return (
+        <Container>
+            <Letter>⌘</Letter>
+            <Letter>K</Letter>
+        </Container>
+    );
+};
diff --git a/datahub-web-react/src/app/search/SearchBar.tsx b/datahub-web-react/src/app/search/SearchBar.tsx
@@ -23,6 +23,7 @@ import { navigateToSearchUrl } from './utils/navigateToSearchUrl';
 import ViewAllSearchItem from './ViewAllSearchItem';
 import { ViewSelect } from '../entity/view/select/ViewSelect';
 import { combineSiblingsInAutoComplete } from './utils/combineSiblingsInAutoComplete';
+import { CommandK } from './CommandK';
 
 const StyledAutoComplete = styled(AutoComplete)`
     width: 100%;
@@ -114,6 +115,7 @@ interface Props {
     fixAutoComplete?: boolean;
     hideRecommendations?: boolean;
     showQuickFilters?: boolean;
+    showCommandK?: boolean;
     viewsEnabled?: boolean;
     combineSiblings?: boolean;
     setIsSearchBarFocused?: (isSearchBarFocused: boolean) => void;
@@ -142,6 +144,7 @@ export const SearchBar = ({
     fixAutoComplete,
     hideRecommendations,
     showQuickFilters,
+    showCommandK = false,
     viewsEnabled = false,
     combineSiblings = false,
     setIsSearchBarFocused,
@@ -153,6 +156,8 @@ export const SearchBar = ({
     const [searchQuery, setSearchQuery] = useState<string | undefined>(initialQuery);
     const [selected, setSelected] = useState<string>();
     const [isDropdownVisible, setIsDropdownVisible] = useState(false);
+    const [isFocused, setIsFocused] = useState(false);
+
     useEffect(() => setSelected(initialQuery), [initialQuery]);
 
     const searchEntityTypes = entityRegistry.getSearchEntityTypes();
@@ -277,11 +282,13 @@ export const SearchBar = ({
     function handleFocus() {
         if (onFocus) onFocus();
         handleSearchBarClick(true);
+        setIsFocused(true);
     }
 
     function handleBlur() {
         if (onBlur) onBlur();
         handleSearchBarClick(false);
+        setIsFocused(false);
     }
 
     function handleSearch(query: string, type?: EntityType, appliedQuickFilters?: FacetFilterInput[]) {
@@ -294,18 +301,21 @@ export const SearchBar = ({
     const searchInputRef = useRef(null);
 
     useEffect(() => {
-        const handleKeyDown = (event) => {
-            // Support command-k to select the search bar.
-            // 75 is the keyCode for 'k'
-            if ((event.metaKey || event.ctrlKey) && event.keyCode === 75) {
-                (searchInputRef?.current as any)?.focus();
-            }
-        };
-        document.addEventListener('keydown', handleKeyDown);
-        return () => {
-            document.removeEventListener('keydown', handleKeyDown);
-        };
-    }, []);
+        if (showCommandK) {
+            const handleKeyDown = (event) => {
+                // Support command-k to select the search bar.
+                // 75 is the keyCode for 'k'
+                if ((event.metaKey || event.ctrlKey) && event.keyCode === 75) {
+                    (searchInputRef?.current as any)?.focus();
+                }
+            };
+            document.addEventListener('keydown', handleKeyDown);
+            return () => {
+                document.removeEventListener('keydown', handleKeyDown);
+            };
+        }
+        return () => null;
+    }, [showCommandK]);
 
     return (
         <AutoCompleteContainer style={style} ref={searchBarWrapperRef}>
@@ -377,7 +387,7 @@ export const SearchBar = ({
                     data-testid="search-input"
                     onFocus={handleFocus}
                     onBlur={handleBlur}
-                    allowClear={{ clearIcon: <ClearIcon /> }}
+                    allowClear={(isFocused && { clearIcon: <ClearIcon /> }) || false}
                     prefix={
                         <>
                             {viewsEnabled && (
@@ -411,6 +421,7 @@ export const SearchBar = ({
                         </>
                     }
                     ref={searchInputRef}
+                    suffix={(showCommandK && !isFocused && <CommandK />) || null}
                 />
             </StyledAutoComplete>
         </AutoCompleteContainer>

diff --git a/datahub-web-react/src/app/search/SearchHeader.tsx b/datahub-web-react/src/app/search/SearchHeader.tsx
@@ -108,6 +108,7 @@ export const SearchHeader = ({
                     fixAutoComplete
                     showQuickFilters
                     showViewAllResults
+                    showCommandK
                 />
             </LogoSearchContainer>
             <NavGroup>