diff --git a/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java b/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java index 21b2697c78..a7d38fad8a 100644 --- a/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java +++ b/data-avro/src/main/java/com/linkedin/data/avro/DataTranslator.java @@ -61,6 +61,18 @@ public class DataTranslator implements DataTranslatorContext { protected DataTranslationOptions _dataTranslationOptions; + public static void backFillMissingDeafultInDataMap(DataMap map, RecordDataSchema dataSchema) { + for (RecordDataSchema.Field field: dataSchema.getFields()) { + DataSchema fieldType = field.getType(); + if ( fieldType.getDereferencedType() != DataSchema.Type.ARRAY && fieldType.getDereferencedType() != DataSchema.Type.MAP) { + continue; + } + if (field.getDefault() != null && (!map.containsKey(field.getName()) || map.get(field.getName()) == null)) { + map.put(field.getName(), field.getDefault()); + } + } + } + /** * Convert the given {@link DataMap} conforming to the provided {@link RecordDataSchema} to a {@link GenericRecord}. * @@ -74,6 +86,7 @@ public class DataTranslator implements DataTranslatorContext */ public static GenericRecord dataMapToGenericRecord(DataMap map, RecordDataSchema dataSchema) throws DataTranslationException { + backFillMissingDeafultInDataMap(map, dataSchema); Schema avroSchema = SchemaTranslator.dataToAvroSchema(dataSchema); return dataMapToGenericRecord(map, dataSchema, avroSchema, null); } @@ -94,6 +107,7 @@ public static GenericRecord dataMapToGenericRecord(DataMap map, RecordDataSchema DataMapToAvroRecordTranslationOptions options) throws DataTranslationException { + backFillMissingDeafultInDataMap(map, dataSchema); Schema avroSchema = SchemaTranslator.dataToAvroSchema(dataSchema); return dataMapToGenericRecord(map, dataSchema, avroSchema, options); } @@ -102,6 +116,7 @@ public static GenericRecord dataMapToGenericRecord(DataMap map, RecordDataSchema DataMapToAvroRecordTranslationOptions options) throws DataTranslationException { + backFillMissingDeafultInDataMap(map, dataSchema); DataMapToGenericRecordTranslator translator = new DataMapToGenericRecordTranslator(options); try { @@ -116,6 +131,7 @@ public static GenericRecord dataMapToGenericRecord(DataMap map, RecordDataSchema public static T dataMapToSpecificRecord(DataMap map, RecordDataSchema dataSchema, Schema avroSchema) throws DataTranslationException { + backFillMissingDeafultInDataMap(map, dataSchema); DataMapToSpecificRecordTranslator translator = new DataMapToSpecificRecordTranslator(); try { T avroRecord = translator.translate(map, dataSchema, avroSchema); @@ -145,6 +161,7 @@ public static T dataMapToSpecificRecord(DataMap m */ public static GenericRecord dataMapToGenericRecord(DataMap map, RecordDataSchema dataSchema, Schema avroSchema) throws DataTranslationException { + backFillMissingDeafultInDataMap(map, dataSchema); return dataMapToGenericRecord(map, dataSchema, avroSchema, null); } diff --git a/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java b/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java index 3abd677853..aaf22eeb5d 100644 --- a/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java +++ b/data-avro/src/test/java/com/linkedin/data/avro/TestDataTranslator.java @@ -17,6 +17,8 @@ package com.linkedin.data.avro; import com.google.common.collect.ImmutableMap; +import com.linkedin.avroutil1.compatibility.AvroCompatibilityHelper; +import com.linkedin.data.Data; import com.linkedin.data.DataList; import com.linkedin.data.DataMap; import com.linkedin.data.TestUtil; @@ -28,6 +30,7 @@ import com.linkedin.data.avro.testevents.RecordArray; import com.linkedin.data.avro.testevents.RecordMap; import com.linkedin.data.avro.testevents.StringRecord; +import com.linkedin.data.avro.testevents.TestArray; import com.linkedin.data.avro.testevents.TestEventRecordOfRecord; import com.linkedin.data.avro.testevents.TestEventWithUnionAndEnum; import com.linkedin.data.avro.util.AvroUtil; @@ -1165,7 +1168,9 @@ private void testDataTranslation(String schemaText, String[][] row) throws IOExc String expectedBeforeNamespaceProcessor = row[col][i]; String expected = TestAvroUtil.namespaceProcessor(expectedBeforeNamespaceProcessor); if (debug && expected != expectedBeforeNamespaceProcessor) out.println(" Expected:" + expected); - + if (!result.contains(expected)) { + System.out.println("RESULT: "+result+"\n"+"EXPECTED: "+expected); + } assertTrue(result.contains(expected)); } @@ -2291,5 +2296,29 @@ public void testMapArrayUnion() throws IOException { Assert.assertTrue(mapOfMapOfArrayOfMapArrayUnion.get(0) instanceof Map); Assert.assertEquals(((Map) mapOfMapOfArrayOfMapArrayUnion.get(0)).get("recordMap"), mapOfArrayOfMapArrayUnion); } + + @Test + public void testDataMapBackfill() throws IOException { + final String SCHEMA = + "{" + + " \"type\":\"record\"," + + " \"name\":\"Foo\"," + + " \"fields\":[" + + " {" + + " \"name\":\"arrayField\"," + + " \"type\":{" + + " \"type\":\"array\"," + + " \"items\":\"string\"" + + " }," + + " \"default\":[ ]" + + " }" + + " ]" + + "}"; + RecordDataSchema recordDataSchema = + (RecordDataSchema) TestUtil.dataSchemaFromString(SCHEMA); + DataMap map = new DataMap(); + GenericRecord record = DataTranslator.dataMapToGenericRecord(map, recordDataSchema); + Assert.assertEquals(record.get("arrayField"), map.get("arrayField")); + } }