Skip to content

Commit

Permalink
test: added test case with avro schema got during Hudi integration
Browse files Browse the repository at this point in the history
  • Loading branch information
@sgomezvillamor
sgomezvillamor committed Dec 11, 2024
1 parent c78c13c commit defce5e
Show file tree
Hide file tree
Showing 2 changed files with 351 additions and 0 deletions.
156 changes: 156 additions & 0 deletions ...b/src/test/java/io/datahubproject/schematron/converters/avro/AvroSchemaConverterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,162 @@ void testLogicalTypes() throws IOException {
"{\"logicalType\":\"uuid\"}");
}

@Test(groups = "basic")
void testUsersRecord() throws IOException {
// this is a test case got during the Hudi integration
SchemaMetadata schema =
avroSchemaConverter.toDataHubSchema(
readAvroSchema("users_record.avsc"), false, false, dataPlatformUrn, null);

schema.getFields().forEach(System.out::println);

assertEquals(schema.getFields().size(), 20);

assertSchemaField(
schema.getFields().get(0),
"[version=2.0].[type=users_record].[type=string]._hoodie_commit_time",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(1),
"[version=2.0].[type=users_record].[type=string]._hoodie_commit_seqno",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(2),
"[version=2.0].[type=users_record].[type=string]._hoodie_record_key",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(3),
"[version=2.0].[type=users_record].[type=string]._hoodie_partition_path",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(4),
"[version=2.0].[type=users_record].[type=string]._hoodie_file_name",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(5),
"[version=2.0].[type=users_record].[type=string].user_id",
"string",
false,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(6),
"[version=2.0].[type=users_record].[type=string].name",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(7),
"[version=2.0].[type=users_record].[type=address].address",
"address",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
assertSchemaField(
schema.getFields().get(8),
"[version=2.0].[type=users_record].[type=address].address.[type=string].street",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(9),
"[version=2.0].[type=users_record].[type=address].address.[type=string].city",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(10),
"[version=2.0].[type=users_record].[type=address].address.[type=string].country",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(11),
"[version=2.0].[type=users_record].[type=address].address.[type=string].postal_code",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(12),
"[version=2.0].[type=users_record].[type=address].address.[type=long].created_at",
"long(timestamp-micros)",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
"{\"logicalType\":\"timestamp-micros\"}");
assertSchemaField(
schema.getFields().get(13),
"[version=2.0].[type=users_record].[type=contact].contact",
"contact",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new RecordType())));
assertSchemaField(
schema.getFields().get(14),
"[version=2.0].[type=users_record].[type=contact].contact.[type=string].email",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(15),
"[version=2.0].[type=users_record].[type=contact].contact.[type=string].phone",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
assertSchemaField(
schema.getFields().get(16),
"[version=2.0].[type=users_record].[type=long].created_at",
"long(timestamp-micros)",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
"{\"logicalType\":\"timestamp-micros\"}");
assertSchemaField(
schema.getFields().get(17),
"[version=2.0].[type=users_record].[type=long].updated_at",
"long(timestamp-micros)",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new TimeType())),
"{\"logicalType\":\"timestamp-micros\"}");
assertSchemaField(
schema.getFields().get(18),
"[version=2.0].[type=users_record].[type=map].[type=int].props",
"int",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new NumberType())));
assertSchemaField(
schema.getFields().get(19),
"[version=2.0].[type=users_record].[type=string].country",
"string",
true,
false,
new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new StringType())));
}

private void assertSchemaField(
SchemaField field,
String expectedPath,
Expand Down
195 changes: 195 additions & 0 deletions metadata-integration/java/datahub-schematron/lib/src/test/resources/users_record.avsc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
{
"type": "record",
"name": "users_record",
"namespace": "hoodie.users",
"fields": [
{
"name": "_hoodie_commit_time",
"type": [
"null",
"string"
],
"doc": "",
"default": null
},
{
"name": "_hoodie_commit_seqno",
"type": [
"null",
"string"
],
"doc": "",
"default": null
},
{
"name": "_hoodie_record_key",
"type": [
"null",
"string"
],
"doc": "",
"default": null
},
{
"name": "_hoodie_partition_path",
"type": [
"null",
"string"
],
"doc": "",
"default": null
},
{
"name": "_hoodie_file_name",
"type": [
"null",
"string"
],
"doc": "",
"default": null
},
{
"name": "user_id",
"type": "string"
},
{
"name": "name",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "address",
"type": [
"null",
{
"type": "record",
"name": "address",
"namespace": "hoodie.users.users_record",
"fields": [
{
"name": "street",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "city",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "country",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "postal_code",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "created_at",
"type": [
"null",
{
"type": "long",
"logicalType": "timestamp-micros"
}
],
"default": null
}
]
}
],
"default": null
},
{
"name": "contact",
"type": [
"null",
{
"type": "record",
"name": "contact",
"namespace": "hoodie.users.users_record",
"fields": [
{
"name": "email",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "phone",
"type": [
"null",
"string"
],
"default": null
}
]
}
],
"default": null
},
{
"name": "created_at",
"type": [
"null",
{
"type": "long",
"logicalType": "timestamp-micros"
}
],
"default": null
},
{
"name": "updated_at",
"type": [
"null",
{
"type": "long",
"logicalType": "timestamp-micros"
}
],
"default": null
},
{
"name": "props",
"type": [
"null",
{
"type": "map",
"values": [
"null",
"int"
]
}
],
"default": null
},
{
"name": "country",
"type": [
"null",
"string"
],
"default": null
}
]
}

0 comments on commit defce5e

Please sign in to comment.