Skip to content

Commit

Permalink
Merge branch 'master' into feat(ingestion/neo4j)
Browse files Browse the repository at this point in the history
  • Loading branch information
k-bartlett authored Nov 26, 2024
2 parents fc0d90f + 094433c commit ea69856
Show file tree
Hide file tree
Showing 96 changed files with 6,965 additions and 1,856 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import com.linkedin.datahub.graphql.generated.Domain;
import com.linkedin.datahub.graphql.generated.ERModelRelationship;
import com.linkedin.datahub.graphql.generated.ERModelRelationshipProperties;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.datahub.graphql.generated.EntityPath;
import com.linkedin.datahub.graphql.generated.EntityRelationship;
import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
Expand Down Expand Up @@ -312,6 +313,7 @@
import com.linkedin.datahub.graphql.resolvers.type.HyperParameterValueTypeResolver;
import com.linkedin.datahub.graphql.resolvers.type.PlatformSchemaUnionTypeResolver;
import com.linkedin.datahub.graphql.resolvers.type.PropertyValueResolver;
import com.linkedin.datahub.graphql.resolvers.type.ResolvedActorResolver;
import com.linkedin.datahub.graphql.resolvers.type.ResultsTypeResolver;
import com.linkedin.datahub.graphql.resolvers.type.TimeSeriesAspectInterfaceTypeResolver;
import com.linkedin.datahub.graphql.resolvers.user.CreateNativeUserResetTokenResolver;
Expand Down Expand Up @@ -1730,12 +1732,22 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
.type(
"InstitutionalMemoryMetadata",
typeWiring ->
typeWiring.dataFetcher(
"author",
new LoadableTypeResolver<>(
corpUserType,
(env) ->
((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())))
typeWiring
.dataFetcher(
"author",
new LoadableTypeResolver<>(
corpUserType,
(env) ->
((InstitutionalMemoryMetadata) env.getSource())
.getAuthor()
.getUrn()))
.dataFetcher(
"actor",
new EntityTypeResolver(
this.entityTypes,
(env) ->
(Entity)
((InstitutionalMemoryMetadata) env.getSource()).getActor())))
.type(
"DatasetStatsSummary",
typeWiring ->
Expand Down Expand Up @@ -2242,6 +2254,7 @@ private void configureTypeResolvers(final RuntimeWiring.Builder builder) {
"HyperParameterValueType",
typeWiring -> typeWiring.typeResolver(new HyperParameterValueTypeResolver()))
.type("PropertyValue", typeWiring -> typeWiring.typeResolver(new PropertyValueResolver()))
.type("ResolvedActor", typeWiring -> typeWiring.typeResolver(new ResolvedActorResolver()))
.type("Aspect", typeWiring -> typeWiring.typeResolver(new AspectInterfaceTypeResolver()))
.type(
"TimeSeriesAspect",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,21 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
final DateTime end,
final String title,
final DateInterval interval) {
final DateRange dateRange =
new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));

final DateRange dateRange;

// adjust month to show 1st of month rather than last day of previous month
if (interval == DateInterval.MONTH) {
dateRange =
new DateRange(
String.valueOf(beginning.plusDays(1).getMillis()), // Shift start by 1 day
String.valueOf(end.plusDays(1).getMillis()) // Shift end by 1 day
);
} else {
// week display starting Sundays
dateRange =
new DateRange(String.valueOf(beginning.getMillis()), String.valueOf(end.getMillis()));
}

final List<NamedLine> timeSeriesLines =
_analyticsService.getTimeseriesChart(
Expand All @@ -96,6 +109,7 @@ private TimeSeriesChart getActiveUsersTimeSeriesChart(
ImmutableMap.of(),
Collections.emptyMap(),
Optional.of("browserId"));

return TimeSeriesChart.builder()
.setTitle(title)
.setDateRange(dateRange)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package com.linkedin.datahub.graphql.resolvers.type;

import com.linkedin.datahub.graphql.generated.CorpGroup;
import com.linkedin.datahub.graphql.generated.CorpUser;
import graphql.TypeResolutionEnvironment;
import graphql.schema.GraphQLObjectType;
import graphql.schema.TypeResolver;

public class ResolvedActorResolver implements TypeResolver {

public static final String CORP_USER = "CorpUser";
public static final String CORP_GROUP = "CorpGroup";

@Override
public GraphQLObjectType getType(TypeResolutionEnvironment env) {
if (env.getObject() instanceof CorpUser) {
return env.getSchema().getObjectType(CORP_USER);
} else if (env.getObject() instanceof CorpGroup) {
return env.getSchema().getObjectType(CORP_GROUP);
} else {
throw new RuntimeException(
"Unrecognized object type provided to type resolver, Type:" + env.getObject().toString());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public InstitutionalMemoryMetadata apply(
result.setDescription(input.getDescription()); // deprecated field
result.setLabel(input.getDescription());
result.setAuthor(getAuthor(input.getCreateStamp().getActor().toString()));
result.setActor(ResolvedActorMapper.map(input.getCreateStamp().getActor()));
result.setCreated(AuditStampMapper.map(context, input.getCreateStamp()));
result.setAssociatedUrn(entityUrn.toString());
return result;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package com.linkedin.datahub.graphql.types.common.mappers;

import com.linkedin.common.urn.Urn;
import com.linkedin.datahub.graphql.generated.CorpGroup;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.EntityType;
import com.linkedin.datahub.graphql.generated.ResolvedActor;
import com.linkedin.metadata.Constants;
import javax.annotation.Nonnull;

public class ResolvedActorMapper {

public static final ResolvedActorMapper INSTANCE = new ResolvedActorMapper();

public static ResolvedActor map(@Nonnull final Urn actorUrn) {
return INSTANCE.apply(actorUrn);
}

public ResolvedActor apply(@Nonnull final Urn actorUrn) {
if (actorUrn.getEntityType().equals(Constants.CORP_GROUP_ENTITY_NAME)) {
CorpGroup partialGroup = new CorpGroup();
partialGroup.setUrn(actorUrn.toString());
partialGroup.setType(EntityType.CORP_GROUP);
return partialGroup;
}
CorpUser partialUser = new CorpUser();
partialUser.setUrn(actorUrn.toString());
partialUser.setType(EntityType.CORP_USER);
return (ResolvedActor) partialUser;
}
}
10 changes: 9 additions & 1 deletion datahub-graphql-core/src/main/resources/entity.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -3005,8 +3005,14 @@ type InstitutionalMemoryMetadata {

"""
The author of this metadata
Deprecated! Use actor instead for users or groups.
"""
author: CorpUser!
author: CorpUser! @deprecated(reason: "Use `actor`")

"""
The author of this metadata
"""
actor: ResolvedActor!

"""
An AuditStamp corresponding to the creation of this resource
Expand Down Expand Up @@ -3834,6 +3840,8 @@ enum CorpUserStatus {
ACTIVE
}

union ResolvedActor = CorpUser | CorpGroup

"""
A DataHub User entity, which represents a Person on the Metadata Entity Graph
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,11 @@ private static EntityClient initMockEntityClient(
Mockito.when(
client.searchAcrossEntities(
any(),
Mockito.eq(entityTypes),
Mockito.argThat(
argument ->
argument != null
&& argument.containsAll(entityTypes)
&& entityTypes.containsAll(argument)),
Mockito.eq(query),
Mockito.eq(filter),
Mockito.eq(start),
Expand All @@ -409,7 +413,11 @@ private static void verifyMockEntityClient(
Mockito.verify(mockClient, Mockito.times(1))
.searchAcrossEntities(
any(),
Mockito.eq(entityTypes),
Mockito.argThat(
argument ->
argument != null
&& argument.containsAll(entityTypes)
&& entityTypes.containsAll(argument)),
Mockito.eq(query),
Mockito.eq(filter),
Mockito.eq(start),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,11 @@ private static EntityClient initMockEntityClient(
Mockito.when(
client.searchAcrossEntities(
any(),
Mockito.eq(entityTypes),
Mockito.argThat(
argument ->
argument != null
&& argument.containsAll(entityTypes)
&& entityTypes.containsAll(argument)),
Mockito.eq(query),
Mockito.eq(filter),
Mockito.eq(start),
Expand All @@ -483,7 +487,11 @@ private static void verifyMockEntityClient(
Mockito.verify(mockClient, Mockito.times(1))
.searchAcrossEntities(
any(),
Mockito.eq(entityTypes),
Mockito.argThat(
argument ->
argument != null
&& argument.containsAll(entityTypes)
&& entityTypes.containsAll(argument)),
Mockito.eq(query),
Mockito.eq(filter),
Mockito.eq(start),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,26 @@ public void testStartOfNextWeek() {
Mockito.when(dateUtil.getNow()).thenReturn(setTimeParts(8, false));
assertEqualStartOfNextWeek(dateUtil, 9);
}

// validates logic to display correct dates in MAU chart
@Test
public void testDateAdjustmentsForMonth() {
DateUtil dateUtil = Mockito.spy(DateUtil.class);

Mockito.when(dateUtil.getNow()).thenReturn(new DateTime(2024, 11, 15, 0, 0, 0));

// start date should be next month minus a day
// but we want to display Dec 1 instead of Nov 30, so add a day and verify it's Dec
DateTime startOfNextMonthMinus12 = dateUtil.getStartOfNextMonth().minusMonths(12);
DateTime adjustedStart = startOfNextMonthMinus12.minusMillis(1).plusDays(1);
assertEquals(12, adjustedStart.getMonthOfYear()); // Verify it is December
assertEquals(2023, adjustedStart.getYear()); // Verify it is 2023

// verify that the end date displays correctly
// the chart will display Oct 1 as the last month because we don't show current month
DateTime startOfThisMonth = dateUtil.getStartOfThisMonth();
DateTime adjustedEnd = startOfThisMonth.minusMillis(1).plusDays(1);
assertEquals(11, adjustedEnd.getMonthOfYear()); // Verify it is November
assertEquals(2024, adjustedEnd.getYear()); // Verify it is 2024
}
}
6 changes: 6 additions & 0 deletions datahub-web-react/src/Mocks.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,12 @@ export const dataset3 = {
username: 'datahub',
type: EntityType.CorpUser,
},
actor: {
__typename: 'CorpUser',
urn: 'urn:li:corpuser:datahub',
username: 'datahub',
type: EntityType.CorpUser,
},
description: 'This only points to Google',
label: 'This only points to Google',
created: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export default function LinkButton({ link }: Props) {
href={link.url}
target="_blank"
rel="noreferrer"
key={`${link.label}-${link.url}-${link.author}`}
key={`${link.label}-${link.url}-${link.actor.urn}`}
>
<LinkOutlined />
{link.description || link.label}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Link } from 'react-router-dom';
import styled from 'styled-components/macro';
import { message, Button, List, Typography, Modal, Form, Input } from 'antd';
import { LinkOutlined, DeleteOutlined, EditOutlined } from '@ant-design/icons';
import { EntityType, InstitutionalMemoryMetadata } from '../../../../../../types.generated';
import { InstitutionalMemoryMetadata } from '../../../../../../types.generated';
import { useEntityData, useMutationUrn } from '../../../EntityContext';
import { useEntityRegistry } from '../../../../../useEntityRegistry';
import { ANTD_GRAY } from '../../../constants';
Expand Down Expand Up @@ -182,10 +182,8 @@ export const LinkList = ({ refetch }: LinkListProps) => {
description={
<>
Added {formatDateString(link.created.time)} by{' '}
<Link
to={`${entityRegistry.getEntityUrl(EntityType.CorpUser, link.author.urn)}`}
>
{link.author.username}
<Link to={`${entityRegistry.getEntityUrl(link.actor.type, link.actor.urn)}`}>
{entityRegistry.getDisplayName(link.actor.type, link.actor)}
</Link>
</>
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@
"name": "csv-enricher",
"displayName": "CSV",
"description": "Import metadata from a formatted CSV.",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv'",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher",
"recipe": "source: \n type: csv-enricher \n config: \n # URL of your csv file to ingest \n filename: \n array_delimiter: '|' \n delimiter: ',' \n write_semantics: PATCH"
},
{
Expand Down
2 changes: 1 addition & 1 deletion datahub-web-react/src/app/ingest/source/conf/csv/csv.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ const csvConfig: SourceConfig = {
type: 'csv-enricher',
placeholderRecipe,
displayName: 'CSV',
docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/csv',
docsUrl: 'https://datahubproject.io/docs/generated/ingestion/sources/csv-enricher',
logoUrl: csvLogo,
};

Expand Down
1 change: 1 addition & 0 deletions datahub-web-react/src/graphql-mock/mutationHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ export const updateEntityLink = ({ entity, institutionalMemory }: UpdateEntityLi
description: e.description as string,
label: e.description as string,
author: { urn: e.author, username: '', type: EntityType.CorpUser },
actor: { urn: e.author, username: '', type: EntityType.CorpUser },
created: { time: Date.now(), actor: getActor(), __typename: 'AuditStamp' },
associatedUrn: dataEntity.urn,
};
Expand Down
5 changes: 2 additions & 3 deletions datahub-web-react/src/graphql/domain.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ query getDomain($urn: String!) {
institutionalMemory {
elements {
url
author {
urn
username
actor {
...resolvedActorFields
}
description
created {
Expand Down
16 changes: 13 additions & 3 deletions datahub-web-react/src/graphql/fragments.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -202,12 +202,22 @@ fragment embedFields on Embed {
renderUrl
}

fragment resolvedActorFields on ResolvedActor {
... on CorpUser {
urn
...entityDisplayNameFields
}
... on CorpGroup {
urn
...entityDisplayNameFields
}
}

fragment institutionalMemoryFields on InstitutionalMemory {
elements {
url
author {
urn
username
actor {
...resolvedActorFields
}
description
created {
Expand Down
2 changes: 2 additions & 0 deletions docker/profiles/docker-compose.gms.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ x-datahub-gms-service: &datahub-gms-service
<<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-search_config.yaml}
ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true}
healthcheck:
test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health
start_period: 90s
Expand Down Expand Up @@ -183,6 +184,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service
environment: &datahub-mce-consumer-env
<<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true}

x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev
<<: *datahub-mce-consumer-service
Expand Down
13 changes: 7 additions & 6 deletions docs/deploy/environment-vars.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@ DataHub works.

## Feature Flags

| Variable | Default | Unit/Type | Components | Description |
|--------------------------------------------------|---------|-----------|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
| `UI_INGESTION_ENABLED` | `true` | boolean | [`GMS`, `MCE Consumer`] | Enable UI based ingestion. |
| `DATAHUB_ANALYTICS_ENABLED` | `true` | boolean | [`Frontend`, `GMS`] | Collect DataHub usage to populate the analytics dashboard. |
| `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true` | boolean | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Do not wait for the `system-update` to complete before starting. This should typically only be disabled during development. |
| `ER_MODEL_RELATIONSHIP_FEATURE_ENABLED` | `false` | boolean | [`Frontend`, `GMS`] | Enable ER Model Relation Feature that shows Relationships Tab within a Dataset UI. |
| Variable | Default | Unit/Type | Components | Description |
|--------------------------------------------------|----------|-----------|------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|
| `UI_INGESTION_ENABLED` | `true` | boolean | [`GMS`, `MCE Consumer`] | Enable UI based ingestion. |
| `DATAHUB_ANALYTICS_ENABLED` | `true` | boolean | [`Frontend`, `GMS`] | Collect DataHub usage to populate the analytics dashboard. |
| `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true` | boolean | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Do not wait for the `system-update` to complete before starting. This should typically only be disabled during development. |
| `ER_MODEL_RELATIONSHIP_FEATURE_ENABLED` | `false` | boolean | [`Frontend`, `GMS`] | Enable ER Model Relation Feature that shows Relationships Tab within a Dataset UI. |
| `STRICT_URN_VALIDATION_ENABLED` | `false` | boolean | [`GMS`, `MCE Consumer`, `MAE Consumer`] | Enable stricter URN validation logic |


## Ingestion
Expand Down
Loading

0 comments on commit ea69856

Please sign in to comment.